@@ -1,110 +1,163 @@
# include "AssetsManager.hpp"
# include "Common/Abstract.hpp"
# include "sqlite3.h"
# include <cstddef>
# include <filesystem>
# include <fstream>
# include <optional>
# include <utility>
namespace LV : : Client {
CacheDatabase : : CacheDatabase ( const fs : : path & cachePath )
: Path ( cachePath )
AssetsManager : : AssetsManager ( boost : : asio : : io_context & ioc , const fs : : path & cachePath ,
size_t maxCacheDirectorySize , size_t maxLifeTime )
: IAsyncDestructible ( ioc ) , CachePath ( cachePath )
{
int errc = sqlite3_open_v2 ( ( Path / " db.sqlite3 " ) . c_str ( ) , & DB , SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE , nullptr ) ;
if ( errc ) {
MAKE_ERROR ( " Н е " < < ( Path / " db.sqlite3 " ) . c_str ( ) < < " : " < < sqlite3_errmsg ( DB ) ) ;
{
auto lock = Changes . lock ( ) ;
lock - > MaxCacheDatabaseSize = maxCacheDirectorySize ;
lock - > MaxLifeTime = maxLifeTime ;
lock - > MaxChange = true ;
}
const char * sql = R " (
CREATE TABLE IF NOT EXISTS files (
sha256 BLOB ( 32 ) NOT NULL , - -
last_used INT NOT NULL , - - unix timestamp
size INT NOT NULL , - - file size
UNIQUE ( sha256 ) ) ;
) " ;
errc = sqlite3_exec ( DB , sql , nullptr , nullptr , nullptr ) ;
if ( errc ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
if ( ! fs : : exists ( PathFiles ) ) {
LOG . debug ( ) < < " Директория для хранения кеша отсутствует, создаём новую ' " < < CachePath < < ' \' ' ;
fs : : create_directories ( PathFiles ) ;
}
sql = R " (
INSERT OR REPLACE INTO files ( sha256 , last_used , size )
VALUES ( ? , ? , ? ) ;
)" ;
LOG . debug ( ) < < " Открываем базу данных кеша... (инициализация sqlite3) " ;
{
int errc = sqlite3_open_v2 ( PathDatabase . c_str ( ) , & DB , SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE , nullptr ) ;
if ( errc )
MAKE_ERROR ( " Н е " < < PathDatabase . c_str ( ) < < " : " < < sqlite3_errmsg ( DB ) ) ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_INSERT , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
const char * sql = R " (
CREATE TABLE IF NOT EXISTS disk_cache (
sha256 BLOB ( 32 ) NOT NULL , - -
last_used INT NOT NULL , - - unix timestamp
size INT NOT NULL , - - file size
UNIQUE ( sha256 ) ) ;
CREATE INDEX IF NOT EXISTS idx__disk_cache__sha256 ON disk_cache ( sha256 ) ;
CREATE TABLE IF NOT EXISTS inline_cache (
sha256 BLOB ( 32 ) NOT NULL , - -
last_used INT NOT NULL , - - unix timestamp
data BLOB NOT NULL , - - file data
UNIQUE ( sha256 ) ) ;
CREATE INDEX IF NOT EXISTS idx__inline_cache__sha256 ON inline_cache ( sha256 ) ;
) " ;
errc = sqlite3_exec ( DB , sql , nullptr , nullptr , nullptr ) ;
if ( errc ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
INSERT OR REPLACE INTO disk_cache ( sha256 , last_used , size )
VALUES ( ? , ? , ? ) ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_DISK_INSERT , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
UPDATE disk_cache SET last_used = ? WHERE sha256 = ? ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_DISK_UPDATE_TIME , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
DELETE FROM disk_cache WHERE sha256 = ? ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_DISK_REMOVE , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT 1 FROM disk_cache where sha256 = ? ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_DISK_CONTAINS , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT SUM ( size ) FROM disk_cache ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_DISK_SUM , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT COUNT ( * ) FROM disk_cache ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_DISK_COUNT , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
INSERT OR REPLACE INTO inline_cache ( sha256 , last_used , data )
VALUES ( ? , ? , ? ) ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_INLINE_INSERT , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT data inline_cache where sha256 = ? ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_INLINE_GET , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
UPDATE inline_cache SET last_used = ? WHERE sha256 = ? ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_INLINE_UPDATE_TIME , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT SUM ( LENGTH ( data ) ) from inline_cache ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_INLINE_SUM , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT COUNT ( * ) FROM inline_cache ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_INLINE_COUNT , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
}
sql = R " (
UPDATE files SET last_used = ? WHERE sha256 = ? ;
) "
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_UPDATE_TIME , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
DELETE FROM files WHERE sha256 = ? ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_REMOVE , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT sha256 FROM files ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_ALL_HASH , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT SUM ( size ) FROM files ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_SUM , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT sha256 , size FROM files WHERE last_used < ? ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_OLD , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT sha256
FROM files
ORDER BY last_used ASC , size ASC
LIMIT (
SELECT COUNT ( * ) FROM (
SELECT SUM ( size ) OVER ( ORDER BY last_used ASC , size ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) AS running_total
FROM files
ORDER BY last_used ASC , size ASC
) sub
WHERE running_total < = ?
) ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_TO_FREE , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sql = R " (
SELECT COUNT ( * ) FROM files ;
) " ;
if ( sqlite3_prepare_v2 ( DB , sql , - 1 , & STMT_COUNT , nullptr ) ! = SQLITE_OK ) {
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
LOG . debug ( ) < < " Успешно, запускаем поток обработки " ;
OffThread = std : : thread ( & AssetsManager : : readWriteThread , this , AUC . use ( ) ) ;
LOG . info ( ) < < " Инициализировано хранилище кеша: " < < CachePath . c_str ( )
}
CacheDatabase : : ~ CacheDatabase ( ) {
for ( sqlite3_stmt * stmt : { STMT_INSERT , STMT_UPDATE_TIME , STMT_REMOVE , STMT_ALL_HASH , STMT_SUM , STMT_OLD , STMT_TO_FREE , STMT_COUNT } )
AssetsManager : : ~ AssetsManager ( ) {
for ( sqlite3_stmt * stmt : {
STMT_DISK_INSERT , STMT_DISK_UPDATE_TIME , STMT_DISK_REMOVE , STMT_DISK_CONTAINS ,
STMT_DISK_SUM , STMT_DISK_COUNT , STMT_INLINE_INSERT , STMT_INLINE_GET ,
STMT_INLINE_UPDATE_TIME , STMT_INLINE_SUM , STMT_INLINE_COUNT
} )
if ( stmt )
sqlite3_finalize ( stmt ) ;
@@ -112,346 +165,223 @@ CacheDatabase::~CacheDatabase() {
sqlite3_close ( DB ) ;
}
size_t CacheDatabase : : getCacheSize ( ) {
size_t Size ;
if ( sqlite3_step ( STMT_SUM ) ! = SQLITE_ROW ) {
sqlite3_reset ( STMT_SUM ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
Size = sqlite3_column_int64 ( STMT_SUM , 0 ) ;
sqlite3_reset ( STMT_SUM ) ;
return Size ;
}
std : : pair < std : : string , size_t > CacheDatabase : : getAllHash ( ) {
if ( sqlite3_step ( STMT_COUNT ) ! = SQLITE_ROW ) {
sqlite3_reset ( STMT_COUNT ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
size_t count = sqlite3_column_int ( STMT_COUNT , 0 ) ;
sqlite3_reset ( STMT_COUNT ) ;
std : : string out ;
out . reserve ( 32 * count ) ;
int errc ;
size_t readed = 0 ;
while ( true ) {
errc = sqlite3_step ( STMT_ALL_HASH ) ;
if ( errc = = SQLITE_DONE )
break ;
else if ( errc ! = SQLITE_ROW ) {
sqlite3_reset ( STMT_ALL_HASH ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
const char * hash = ( const char * ) sqlite3_column_blob ( STMT_ALL_HASH , 0 ) ;
readed + + ;
out + = std : : string_view ( hash , hash + 32 ) ;
}
sqlite3_reset ( STMT_ALL_HASH ) ;
return { out , readed } ;
}
void CacheDatabase : : updateTimeFor ( Hash_t hash ) {
sqlite3_bind_blob ( STMT_UPDATE_TIME , 1 , ( const void * ) hash . data ( ) , 32 , SQLITE_STATIC ) ;
sqlite3_bind_int ( STMT_UPDATE_TIME , 2 , time ( nullptr ) ) ;
if ( sqlite3_step ( STMT_UPDATE_TIME ) ! = SQLITE_DONE ) {
sqlite3_reset ( STMT_UPDATE_TIME ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sqlite3_reset ( STMT_UPDATE_TIME ) ;
}
void CacheDatabase : : insert ( Hash_t hash , size_t size ) {
assert ( size < ( size_t ( 1 ) < < 31 ) - 1 & & size > 0 ) ;
sqlite3_bind_blob ( STMT_INSERT , 1 , ( const void * ) hash . data ( ) , 32 , SQLITE_STATIC ) ;
sqlite3_bind_int ( STMT_INSERT , 2 , time ( nullptr ) ) ;
sqlite3_bind_int ( STMT_INSERT , 3 , ( int ) size ) ;
if ( sqlite3_step ( STMT_INSERT ) ! = SQLITE_DONE ) {
sqlite3_reset ( STMT_INSERT ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sqlite3_reset ( STMT_INSERT ) ;
}
std : : vector < Hash_t > CacheDatabase : : findExcessHashes ( size_t bytesToFree , int timeBefore = time ( nullptr ) - 604800 ) {
std : : vector < Hash_t > out ;
size_t removed = 0 ;
sqlite3_bind_int ( STMT_OLD , 1 , timeBefore ) ;
while ( true ) {
int errc = sqlite3_step ( STMT_OLD ) ;
if ( errc = = SQLITE_DONE )
break ;
else if ( errc ! = SQLITE_ROW ) {
sqlite3_reset ( STMT_OLD ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
const uint8_t * hash = ( const uint8_t * ) sqlite3_column_blob ( STMT_OLD , 0 ) ;
removed + = sqlite3_column_int ( STMT_OLD , 1 ) ;
Hash_t obj ;
for ( int iter = 0 ; iter < 32 ; iter + + )
obj [ iter ] = hash [ iter ] ;
out . push_back ( obj ) ;
}
sqlite3_reset ( STMT_OLD ) ;
if ( removed > bytesToFree )
return out ;
sqlite3_bind_int ( STMT_TO_FREE , 1 , ( int ) bytesToFree ) ;
while ( true ) {
int errc = sqlite3_step ( STMT_TO_FREE ) ;
if ( errc = = SQLITE_DONE )
break ;
else if ( errc ! = SQLITE_ROW ) {
sqlite3_reset ( STMT_TO_FREE ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
const uint8_t * hash = ( const uint8_t * ) sqlite3_column_blob ( STMT_TO_FREE , 0 ) ;
Hash_t obj ;
for ( int iter = 0 ; iter < 32 ; iter + + )
obj [ iter ] = hash [ iter ] ;
out . push_back ( obj ) ;
}
sqlite3_reset ( STMT_TO_FREE ) ;
return out ;
}
void CacheDatabase : : remove ( Hash_t hash ) {
sqlite3_bind_blob ( STMT_REMOVE , 1 , ( const void * ) hash . data ( ) , 32 , SQLITE_STATIC ) ;
if ( sqlite3_step ( STMT_REMOVE ) ! = SQLITE_DONE ) {
sqlite3_reset ( STMT_REMOVE ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sqlite3_reset ( STMT_REMOVE ) ;
}
std : : string CacheDatabase : : hashToString ( Hash_t hash ) {
std : : string text ;
text . reserve ( 64 ) ;
for ( int iter = 0 ; iter < 32 ; iter + + ) {
int val = ( hash [ 31 - iter ] > > 4 ) & 0xf ;
if ( val > 9 )
text + = ' a ' + val - 10 ;
else
text + = ' 0 ' + val ;
val = hash [ 31 - iter ] & 0xf ;
if ( val > 9 )
text + = ' a ' + val - 10 ;
else
text + = ' 0 ' + val ;
}
return text ;
}
// int CacheDatabase::hexCharToInt(char c) {
// if (c >= '0' && c <= '9') return c - '0';
// if (c >= 'a' && c <= 'f') return c - 'a' + 10;
// throw std::invalid_argument("Invalid hexadecimal character");
// }
// Hash_t CacheDatabase::stringToHash(const std::string_view view) {
// if (view.size() != 64)
// throw std::invalid_argument("Hex string must be exactly 64 characters long");
// Hash_t hash;
// for (size_t i = 0; i < 32; ++i) {
// size_t offset = 62 - i * 2;
// int high = hexCharToInt(view[offset]);
// int low = hexCharToInt(view[offset + 1]);
// hash[i] = (high << 4) | low;
// }
// return hash;
// }
coro < > AssetsManager : : asyncDestructor ( ) {
assert ( NeedShutdown ) ; // Нормальный shutdown должен быть вызван
assert ( NeedShutdown ) ; // Должен быть вызван нормальный shutdown
co_await IAsyncDestructible : : asyncDestructor ( ) ;
}
void AssetsManager : : readWriteThread ( AsyncUseControl : : Lock lock ) {
LOG . info ( ) < < " Поток чтения/записи запущен " ;
while ( ! NeedShutdown | | ! WriteQueue . get_read ( ) . empty ( ) ) {
if ( ! ReadQueue . get_read ( ) . empty ( ) ) {
auto lock = ReadQueue . lock ( ) ;
if ( ! lock - > empty ( ) ) {
Hash_t hash = lock - > front ( ) ;
lock - > pop ( ) ;
lock . unlock ( ) ;
try {
std : : vector < fs : : path > assets ;
size_t maxCacheDatabaseSize , maxLifeTime ;
std : : string name = CacheDatabase : : hashToString ( hash ) ;
fs : : path path = Path / name . substr ( 0 , 2 ) / name . substr ( 2 , 2 ) / name . substr ( 4 ) ;
while ( ! NeedShutdown & & ! WriteQueue . get_read ( ) . empty ( ) ) {
// Получить новые данные
if ( Changes . get_read ( ) . AssetsChange ) {
auto lock = Changes . lock ( ) ;
assets = std : : move ( lock - > Assets ) ;
lock - > AssetsChange = false ;
}
std : : s hared_ptr < std : : string > data ;
if ( C hanges . get_read ( ) . MaxChange ) {
auto lock = Changes . lock ( ) ;
maxCacheDatabaseSize = lock - > MaxCacheDatabaseSize ;
maxLifeTime = lock - > MaxLifeTime ;
lock - > MaxChange = false ;
}
if ( Changes . get_read ( ) . FullRecheck ) {
std : : move_only_function < void ( std : : string ) > onRecheckEnd ;
{
auto lock_wc = WriteCache . lock ( ) ;
auto iter = lock_wc - > begin ( ) ;
while ( iter ! = lock_wc - > end ( ) ) {
if ( iter - > first = = hash ) {
// Копируем
data = std : : make_shared < std : : string > ( * iter - > second ) ;
break ;
}
auto lock = Changes . lock ( ) ;
onRecheckEnd = std : : move ( * lock - > OnRecheckEnd ) ;
lock - > FullRecheck = false ;
}
LOG . info ( ) < < " Начата проверка консистентности кеша ассетов " ;
LOG . info ( ) < < " Завершена проверка консистентности кеша ассетов " ;
}
// Чтение
if ( ! ReadQueue . get_read ( ) . empty ( ) ) {
ResourceKey rk ;
{
auto lock = ReadQueue . lock ( ) ;
rk = lock - > front ( ) ;
lock - > pop ( ) ;
}
bool finded = false ;
// Сначала пробежимся по ресурспакам
{
std : : string_view type ;
switch ( rk . Type ) {
case EnumAssets : : Nodestate : type = " nodestate " ; break ;
case EnumAssets : : Particle : type = " particle " ; break ;
case EnumAssets : : Animation : type = " animation " ; break ;
case EnumAssets : : Model : type = " model " ; break ;
case EnumAssets : : Texture : type = " texture " ; break ;
case EnumAssets : : Sound : type = " sound " ; break ;
case EnumAssets : : Font : type = " font " ; break ;
default :
std : : unreachable ( ) ;
}
for ( const fs : : path & path : assets ) {
fs : : path end = path / rk . Domain / type / rk . Key ;
if ( ! fs : : exists ( end ) )
continue ;
// Нашли
finded = true ;
Resource res = Resource ( end ) . convertToMem ( ) ;
ReadyQueue . lock ( ) - > emplace_back ( rk . Hash , res ) ;
break ;
}
}
if ( ! data ) {
data = std : : make_shared < std : : string > ( ) ;
try {
std : : ifstream fd ( path , std : : ios : : binary | std : : ios : : ate ) ;
if ( ! fd . is_open ( ) )
MAKE_ERROR ( " !is_open(): " < < fd . exceptions ( ) ) ;
if ( fd . fail ( ) )
MAKE_ERROR ( " fail(): " < < fd . exceptions ( ) ) ;
std : : ifstream : : pos_type size = fd . tellg ( ) ;
fd . seekg ( 0 , std : : ios : : beg ) ;
data - > resize ( size ) ;
fd . read ( data - > data ( ) , size ) ;
if ( ! finded ) {
// Поищем в малой базе
sqlite3_bind_blob ( STMT_INLINE_GET , 1 , ( const void * ) rk . Hash . data ( ) , 32 , SQLITE_STATIC ) ;
int errc = sqlite3_step ( STMT_INLINE_GET ) ;
if ( errc = = SQLITE_ROW ) {
// Есть запись
const uint8_t * hash = ( const uint8_t * ) sqlite3_column_blob ( STMT_INLINE_GET , 0 ) ;
int size = sqlite3_column_bytes ( STMT_INLINE_GET , 0 ) ;
Resource res ( hash , size ) ;
finded = true ;
ReadyQueue . lock ( ) - > emplace_back ( rk . Hash , res ) ;
} else if ( errc ! = SQLITE_DONE ) {
sqlite3_reset ( STMT_INLINE_GET ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
if ( ! fd . good ( ) )
MAKE_ERROR ( " !good(): " < < fd . exceptions ( ) ) ;
sqlite3_reset ( STMT_INLINE_GET ) ;
DB . updateTimeFor ( hash ) ;
} catch ( const std : : exception & exc ) {
LOG . error ( ) < < " Н е р е с у р с " < < path . c_str ( ) < < " : " < < exc . what ( ) ;
}
if ( finded ) {
sqlite3_bind_blob ( STMT_INLINE_UPDATE_TIME , 1 , ( const void * ) rk . Hash . data ( ) , 32 , SQLITE_STATIC ) ;
sqlite3_bind_int ( STMT_INLINE_UPDATE_TIME , 2 , time ( nullptr ) ) ;
if ( sqlite3_step ( STMT_INLINE_UPDATE_TIME ) ! = SQLITE_DONE ) {
sqlite3_reset ( STMT_INLINE_UPDATE_TIME ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sqlite3_reset ( STMT_INLINE_UPDATE_TIME ) ;
}
}
if ( ! finded ) {
// Поищем на диске
sqlite3_bind_blob ( STMT_DISK_CONTAINS , 1 , ( const void * ) rk . Hash . data ( ) , 32 , SQLITE_STATIC ) ;
int errc = sqlite3_step ( STMT_DISK_CONTAINS ) ;
if ( errc = = SQLITE_ROW ) {
// Есть запись
std : : string hashKey ;
{
std : : stringstream ss ;
ss < < std : : hex < < std : : setfill ( ' 0 ' ) < < std : : setw ( 2 ) ;
for ( int i = 0 ; i < 32 ; + + i )
ss < < static_cast < int > ( rk . Hash [ i ] ) ;
hashKey = ss . str ( ) ;
}
finded = true ;
ReadyQueue . lock ( ) - > emplace_back ( rk . Hash , PathFiles / hashKey . substr ( 0 , 2 ) / hashKey . substr ( 2 ) ) ;
} else if ( errc ! = SQLITE_DONE ) {
sqlite3_reset ( STMT_DISK_CONTAINS ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sqlite3_reset ( STMT_DISK_CONTAINS ) ;
if ( finded ) {
sqlite3_bind_blob ( STMT_DISK_CONTAINS , 1 , ( const void * ) rk . Hash . data ( ) , 32 , SQLITE_STATIC ) ;
sqlite3_bind_int ( STMT_DISK_CONTAINS , 2 , time ( nullptr ) ) ;
if ( sqlite3_step ( STMT_DISK_CONTAINS ) ! = SQLITE_DONE ) {
sqlite3_reset ( STMT_DISK_CONTAINS ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sqlite3_reset ( STMT_DISK_CONTAINS ) ;
}
}
if ( ! finded ) {
// Н е
ReadyQueue . lock ( ) - > emplace_back ( rk . Hash , std : : nullopt ) ;
}
continue ;
}
// Запись
if ( ! WriteQueue . get_read ( ) . empty ( ) ) {
Resource res ;
{
auto lock = WriteQueue . lock ( ) ;
res = lock - > front ( ) ;
lock - > pop ( ) ;
}
// TODO: добавить вычистку места при нехватке
if ( res . size ( ) < = SMALL_RESOURCE ) {
sqlite3_bind_blob ( STMT_INLINE_INSERT , 1 , ( const void * ) res . hash ( ) . data ( ) , 32 , SQLITE_STATIC ) ;
sqlite3_bind_int ( STMT_INLINE_INSERT , 2 , time ( nullptr ) ) ;
sqlite3_bind_blob ( STMT_INLINE_INSERT , 3 , res . data ( ) , res . size ( ) , SQLITE_STATIC ) ;
if ( sqlite3_step ( STMT_INLINE_INSERT ) ! = SQLITE_DONE ) {
sqlite3_reset ( STMT_INLINE_INSERT ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sqlite3_reset ( STMT_INLINE_INSERT ) ;
} else {
std : : string hashKey ;
{
std : : stringstream ss ;
ss < < std : : hex < < std : : setfill ( ' 0 ' ) < < std : : setw ( 2 ) ;
for ( int i = 0 ; i < 32 ; + + i )
ss < < static_cast < int > ( res . hash ( ) [ i ] ) ;
hashKey = ss . str ( ) ;
}
fs : : path end = PathFiles / hashKey . substr ( 0 , 2 ) / hashKey . substr ( 2 ) ;
std : : ofstream fd ( end , std : : ios : : binary ) ;
fd . write ( ( const char * ) res . data ( ) , res . size ( ) ) ;
if ( fd . fail ( ) )
MAKE_ERROR ( " Ошибка записи в файл: " < < end . string ( ) ) ;
fd . close ( ) ;
sqlite3_bind_blob ( STMT_DISK_INSERT , 1 , ( const void * ) res . hash ( ) . data ( ) , 32 , SQLITE_STATIC ) ;
sqlite3_bind_int ( STMT_DISK_INSERT , 2 , time ( nullptr ) ) ;
sqlite3_bind_int ( STMT_DISK_INSERT , 3 , res . size ( ) ) ;
if ( sqlite3_step ( STMT_DISK_INSERT ) ! = SQLITE_DONE ) {
sqlite3_reset ( STMT_DISK_INSERT ) ;
MAKE_ERROR ( " Н е " < < sqlite3_errmsg ( DB ) ) ;
}
sqlite3_reset ( STMT_DISK_INSERT ) ;
}
ReadedQueue . lock ( ) - > emplace_back ( hash , std : : move ( data ) ) ;
continue ;
}
}
if ( ! WriteQueue . get_read ( ) . empty ( ) ) {
auto lock = WriteQueue . lock ( ) ;
if ( ! lock - > empty ( ) ) {
DataTask task = lock - > front ( ) ;
lock - > pop ( ) ;
lock . unlock ( ) ;
std : : string name = CacheDatabase : : hashToString ( task . Hash ) ;
fs : : path path = Path / name . substr ( 0 , 2 ) / name . substr ( 2 , 2 ) / name . substr ( 4 ) ;
try {
// Проверка на наличие свободного места (виртуально)
if ( ssize_t free = ssize_t ( MaxCacheDirectorySize ) - DB . getCacheSize ( ) ; free < task . Data - > size ( ) ) {
// Недостаточно места, сколько необходимо освободить с
ssize_t need = task . Data - > size ( ) - free + 64 * 1024 * 1024 ;
std : : vector < Hash_t > hashes = DB . findExcessHashes ( need , time ( nullptr ) - MaxLifeTime ) ;
LOG . warn ( ) < < " Удаление устаревшего кеша в количестве " < < hashes . size ( ) < < " ... " ;
for ( Hash_t hash : hashes ) {
std : : string name = CacheDatabase : : hashToString ( hash ) ;
fs : : path path = Path / name . substr ( 0 , 2 ) / name . substr ( 2 , 2 ) / name . substr ( 4 ) ;
DB . remove ( hash ) ;
fs : : remove ( path ) ;
fs : : path up1 = path . parent_path ( ) ;
LOG . info ( ) < < " В " < < up1 . c_str ( ) < < " не осталось файлов, удаляем... " ;
size_t count = std : : distance ( fs : : directory_iterator ( up1 ) , fs : : directory_iterator ( ) ) ;
if ( count = = 0 )
fs : : remove ( up1 ) ;
}
}
fs : : create_directories ( path . parent_path ( ) ) ;
std : : ofstream fd ( path , std : : ios : : binary | std : : ios : : ate ) ;
fd . write ( task . Data - > data ( ) , task . Data - > size ( ) ) ;
DB . insert ( task . Hash , task . Data - > size ( ) ) ;
} catch ( const std : : exception & exc ) {
LOG . error ( ) < < " Н е р е с у р с " < < path . c_str ( ) < < " : " < < exc . what ( ) ;
}
auto lock = WriteCache . lock ( ) ;
auto iter = lock - > begin ( ) ;
while ( iter ! = lock - > end ( ) ) {
if ( iter - > first = = task . Hash )
break ;
iter + + ;
}
assert ( iter ! = lock - > end ( ) ) ;
lock - > erase ( iter ) ;
}
}
TOS : : Time : : sleep3 ( 20 ) ;
} catch ( const std : : exception & exc ) {
LOG . warn ( ) < < " Ошибка в работе потока: " < < exc . what ( ) ;
IssuedAnError = true ;
}
LOG . info ( ) < < " Поток чтения/записи остановлен " ;
lock . unlock ( ) ;
}
AssetsManager : : AssetsManager ( boost : : asio : : io_context & ioc , const fs : : path & cachePath ,
size_t maxCacheDirectorySize , size_t maxLifeTime )
: IAsyncDestructible ( ioc ) ,
OffThread ( & AssetsManager : : readWriteThread , this , AUC . use ( ) )
{
LOG . info ( ) < < " Инициализировано хранилище кеша: " < < cachePath . c_str ( ) ;
}
// void ResourceHandler::updateParams(size_t maxLifeTime, size_t maxCacheDirectorySize) {
// MaxLifeTime = maxLifeTime;
// if(MaxCacheDirectorySize != maxCacheDirectorySize) {
// MaxCacheDirectorySize = maxCacheDirectorySize;
// size_t size = DB.getCacheSize();
// if(size > maxCacheDirectorySize) {
// size_t needToFree = size-maxCacheDirectorySize+64*1024*1024;
// try {
// LOG.info() << "Начата вычистка кеша на сумму " << needToFree/1024/1024 << " М б
// std::vector<Hash_t> hashes = DB.findExcessHashes(needToFree, time(nullptr)-MaxLifeTime);
// LOG.warn() << "Удаление кеша в количестве " << hashes.size() << "...";
// for(Hash_t hash : hashes) {
// std::string name = CacheDatabase::hashToString(hash);
// fs::path path = Path / name.substr(0, 2) / name.substr(2, 2) / name.substr(4);
// DB.remove(hash);
// fs::remove(path);
// fs::path up1 = path.parent_path();
// LOG.info() << "В
// size_t count = std::distance(fs::directory_iterator(up1), fs::directory_iterator());
// if(count == 0)
// fs::remove(up1);
// }
// } catch(const std::exception &exc) {
// LOG.error() << "Н е
// }
// }
// }
// }
}
