From 6c7a6df8f64dc2743ac39f771b118b705cf0b1af Mon Sep 17 00:00:00 2001 From: DrSocalkwe3n Date: Mon, 5 Jan 2026 01:50:17 +0600 Subject: [PATCH] =?UTF-8?q?codex-5.2:=20=D0=BF=D0=B5=D1=80=D0=B5=D1=81?= =?UTF-8?q?=D1=82=D1=80=D0=BE=D0=B9=D0=BA=D0=B0=20Client/AssetsManager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Src/Client/AssetsHeaderCodec.cpp | 484 +++++++++++++++++ Src/Client/AssetsHeaderCodec.hpp | 27 + Src/Client/AssetsManager.cpp | 855 ++++++++++--------------------- Src/Client/AssetsManager.hpp | 175 ++++++- Src/Common/AssetsPreloader.cpp | 4 +- docs/assets_manager.md | 66 +++ 6 files changed, 1015 insertions(+), 596 deletions(-) create mode 100644 Src/Client/AssetsHeaderCodec.cpp create mode 100644 Src/Client/AssetsHeaderCodec.hpp create mode 100644 docs/assets_manager.md diff --git a/Src/Client/AssetsHeaderCodec.cpp b/Src/Client/AssetsHeaderCodec.cpp new file mode 100644 index 0000000..c5b03dd --- /dev/null +++ b/Src/Client/AssetsHeaderCodec.cpp @@ -0,0 +1,484 @@ +#include "Client/AssetsHeaderCodec.hpp" +#include +#include +#include "TOSLib.hpp" + +namespace LV::Client::AssetsHeaderCodec { + +namespace { + +struct ParsedModelHeader { + std::vector ModelDeps; + std::vector> TexturePipelines; + std::vector TextureDeps; +}; + +std::optional> parseNodestateHeaderBytes(const std::vector& header) { + if(header.empty() || header.size() % sizeof(ResourceId) != 0) + return std::nullopt; + + const size_t count = header.size() / sizeof(ResourceId); + std::vector deps; + deps.resize(count); + for(size_t i = 0; i < count; ++i) { + ResourceId raw = 0; + std::memcpy(&raw, header.data() + i * sizeof(ResourceId), sizeof(ResourceId)); + deps[i] = raw; + } + return deps; +} + +struct PipelineRemapResult { + bool Ok = true; + std::string Error; +}; + +PipelineRemapResult remapTexturePipelineIds(std::vector& code, + const std::function& mapId) +{ + struct Range { + size_t Start = 0; + size_t End = 0; + }; + + enum class SrcKind : uint8_t { TexId = 0, Sub = 1 }; + enum class Op : uint8_t { + End = 0, + Base_Tex = 1, + Base_Fill = 2, + Base_Anim = 3, + Resize = 10, + Transform = 11, + Opacity = 12, + NoAlpha = 13, + MakeAlpha = 14, + Invert = 15, + Brighten = 16, + Contrast = 17, + Multiply = 18, + Screen = 19, + Colorize = 20, + Anim = 21, + Overlay = 30, + Mask = 31, + LowPart = 32, + Combine = 40 + }; + + struct SrcMeta { + SrcKind Kind = SrcKind::TexId; + uint32_t TexId = 0; + uint32_t Off = 0; + uint32_t Len = 0; + size_t TexIdOffset = 0; + }; + + const size_t size = code.size(); + std::vector visited; + + auto read8 = [&](size_t& ip, uint8_t& out)->bool{ + if(ip >= size) + return false; + out = code[ip++]; + return true; + }; + auto read16 = [&](size_t& ip, uint16_t& out)->bool{ + if(ip + 1 >= size) + return false; + out = uint16_t(code[ip]) | (uint16_t(code[ip + 1]) << 8); + ip += 2; + return true; + }; + auto read24 = [&](size_t& ip, uint32_t& out)->bool{ + if(ip + 2 >= size) + return false; + out = uint32_t(code[ip]) + | (uint32_t(code[ip + 1]) << 8) + | (uint32_t(code[ip + 2]) << 16); + ip += 3; + return true; + }; + auto read32 = [&](size_t& ip, uint32_t& out)->bool{ + if(ip + 3 >= size) + return false; + out = uint32_t(code[ip]) + | (uint32_t(code[ip + 1]) << 8) + | (uint32_t(code[ip + 2]) << 16) + | (uint32_t(code[ip + 3]) << 24); + ip += 4; + return true; + }; + + auto readSrc = [&](size_t& ip, SrcMeta& out)->bool{ + uint8_t kind = 0; + if(!read8(ip, kind)) + return false; + out.Kind = static_cast(kind); + if(out.Kind == SrcKind::TexId) { + out.TexIdOffset = ip; + return read24(ip, out.TexId); + } + if(out.Kind == SrcKind::Sub) { + return read24(ip, out.Off) && read24(ip, out.Len); + } + return false; + }; + + auto patchTexId = [&](const SrcMeta& src)->PipelineRemapResult{ + if(src.Kind != SrcKind::TexId) + return {}; + uint32_t newId = mapId(src.TexId); + if(newId >= (1u << 24)) + return {false, "TexId exceeds u24 range"}; + if(src.TexIdOffset + 2 >= code.size()) + return {false, "TexId patch outside pipeline"}; + code[src.TexIdOffset + 0] = uint8_t(newId & 0xFFu); + code[src.TexIdOffset + 1] = uint8_t((newId >> 8) & 0xFFu); + code[src.TexIdOffset + 2] = uint8_t((newId >> 16) & 0xFFu); + return {}; + }; + + std::function scan; + scan = [&](size_t start, size_t end) -> bool { + if(start >= end || end > size) + return true; + for(const auto& range : visited) { + if(range.Start == start && range.End == end) + return true; + } + visited.push_back(Range{start, end}); + + size_t ip = start; + while(ip < end) { + uint8_t opByte = 0; + if(!read8(ip, opByte)) + return false; + Op op = static_cast(opByte); + switch(op) { + case Op::End: + return true; + + case Op::Base_Tex: { + SrcMeta src{}; + if(!readSrc(ip, src)) + return false; + PipelineRemapResult r = patchTexId(src); + if(!r.Ok) + return false; + if(src.Kind == SrcKind::Sub) { + size_t subStart = src.Off; + size_t subEnd = subStart + src.Len; + if(!scan(subStart, subEnd)) + return false; + } + } break; + + case Op::Base_Fill: { + uint16_t tmp16 = 0; + uint32_t tmp32 = 0; + if(!read16(ip, tmp16)) return false; + if(!read16(ip, tmp16)) return false; + if(!read32(ip, tmp32)) return false; + } break; + + case Op::Base_Anim: { + SrcMeta src{}; + if(!readSrc(ip, src)) return false; + PipelineRemapResult r = patchTexId(src); + if(!r.Ok) return false; + uint16_t frameW = 0; + uint16_t frameH = 0; + uint16_t frameCount = 0; + uint16_t fpsQ = 0; + uint8_t flags = 0; + if(!read16(ip, frameW)) return false; + if(!read16(ip, frameH)) return false; + if(!read16(ip, frameCount)) return false; + if(!read16(ip, fpsQ)) return false; + if(!read8(ip, flags)) return false; + (void)frameW; (void)frameH; (void)frameCount; (void)fpsQ; (void)flags; + if(src.Kind == SrcKind::Sub) { + size_t subStart = src.Off; + size_t subEnd = subStart + src.Len; + if(!scan(subStart, subEnd)) return false; + } + } break; + + case Op::Resize: { + uint16_t tmp16 = 0; + if(!read16(ip, tmp16)) return false; + if(!read16(ip, tmp16)) return false; + } break; + + case Op::Transform: + case Op::Opacity: + case Op::Invert: + if(!read8(ip, opByte)) return false; + break; + + case Op::NoAlpha: + case Op::Brighten: + break; + + case Op::MakeAlpha: + if(ip + 2 >= size) return false; + ip += 3; + break; + + case Op::Contrast: + if(ip + 1 >= size) return false; + ip += 2; + break; + + case Op::Multiply: + case Op::Screen: { + uint32_t tmp32 = 0; + if(!read32(ip, tmp32)) return false; + } break; + + case Op::Colorize: { + uint32_t tmp32 = 0; + if(!read32(ip, tmp32)) return false; + if(!read8(ip, opByte)) return false; + } break; + + case Op::Anim: { + uint16_t frameW = 0; + uint16_t frameH = 0; + uint16_t frameCount = 0; + uint16_t fpsQ = 0; + uint8_t flags = 0; + if(!read16(ip, frameW)) return false; + if(!read16(ip, frameH)) return false; + if(!read16(ip, frameCount)) return false; + if(!read16(ip, fpsQ)) return false; + if(!read8(ip, flags)) return false; + (void)frameW; (void)frameH; (void)frameCount; (void)fpsQ; (void)flags; + } break; + + case Op::Overlay: + case Op::Mask: { + SrcMeta src{}; + if(!readSrc(ip, src)) return false; + PipelineRemapResult r = patchTexId(src); + if(!r.Ok) return false; + if(src.Kind == SrcKind::Sub) { + size_t subStart = src.Off; + size_t subEnd = subStart + src.Len; + if(!scan(subStart, subEnd)) return false; + } + } break; + + case Op::LowPart: { + if(!read8(ip, opByte)) return false; + SrcMeta src{}; + if(!readSrc(ip, src)) return false; + PipelineRemapResult r = patchTexId(src); + if(!r.Ok) return false; + if(src.Kind == SrcKind::Sub) { + size_t subStart = src.Off; + size_t subEnd = subStart + src.Len; + if(!scan(subStart, subEnd)) return false; + } + } break; + + case Op::Combine: { + uint16_t w = 0, h = 0, n = 0; + if(!read16(ip, w)) return false; + if(!read16(ip, h)) return false; + if(!read16(ip, n)) return false; + for(uint16_t i = 0; i < n; ++i) { + uint16_t tmp16 = 0; + if(!read16(ip, tmp16)) return false; + if(!read16(ip, tmp16)) return false; + SrcMeta src{}; + if(!readSrc(ip, src)) return false; + PipelineRemapResult r = patchTexId(src); + if(!r.Ok) return false; + if(src.Kind == SrcKind::Sub) { + size_t subStart = src.Off; + size_t subEnd = subStart + src.Len; + if(!scan(subStart, subEnd)) return false; + } + } + (void)w; (void)h; + } break; + + default: + return false; + } + } + return true; + }; + + if(!scan(0, size)) + return {false, "Invalid texture pipeline bytecode"}; + return {}; +} + +std::vector collectTexturePipelineIds(const std::vector& code) { + std::vector out; + std::unordered_set seen; + + auto addId = [&](uint32_t id) { + if(seen.insert(id).second) + out.push_back(id); + }; + + std::vector copy = code; + auto result = remapTexturePipelineIds(copy, [&](uint32_t id) { + addId(id); + return id; + }); + + if(!result.Ok) + return {}; + + return out; +} + +std::optional parseModelHeaderBytes(const std::vector& header) { + if(header.empty()) + return std::nullopt; + + ParsedModelHeader result; + try { + TOS::ByteBuffer buffer(header.size(), header.data()); + auto reader = buffer.reader(); + + uint16_t modelCount = reader.readUInt16(); + result.ModelDeps.reserve(modelCount); + for(uint16_t i = 0; i < modelCount; ++i) + result.ModelDeps.push_back(reader.readUInt32()); + + uint16_t texCount = reader.readUInt16(); + result.TexturePipelines.reserve(texCount); + for(uint16_t i = 0; i < texCount; ++i) { + uint32_t size32 = reader.readUInt32(); + TOS::ByteBuffer pipe; + reader.readBuffer(pipe); + if(pipe.size() != size32) + return std::nullopt; + result.TexturePipelines.emplace_back(pipe.begin(), pipe.end()); + } + + std::unordered_set seen; + for(const auto& pipe : result.TexturePipelines) { + for(uint32_t id : collectTexturePipelineIds(pipe)) { + if(seen.insert(id).second) + result.TextureDeps.push_back(id); + } + } + } catch(const std::exception&) { + return std::nullopt; + } + + return result; +} + +} // namespace + +std::optional parseHeader(EnumAssets type, const std::vector& header) { + if(header.empty()) + return std::nullopt; + + ParsedHeader result; + result.Type = type; + + if(type == EnumAssets::Nodestate) { + auto deps = parseNodestateHeaderBytes(header); + if(!deps) + return std::nullopt; + result.ModelDeps = std::move(*deps); + return result; + } + + if(type == EnumAssets::Model) { + auto parsed = parseModelHeaderBytes(header); + if(!parsed) + return std::nullopt; + result.ModelDeps = std::move(parsed->ModelDeps); + result.TexturePipelines = std::move(parsed->TexturePipelines); + result.TextureDeps = std::move(parsed->TextureDeps); + return result; + } + + return std::nullopt; +} + +std::vector rebindHeader(EnumAssets type, const std::vector& header, + const MapIdFn& mapModelId, const MapIdFn& mapTextureId, const WarnFn& warn) +{ + if(header.empty()) + return {}; + + if(type == EnumAssets::Nodestate) { + if(header.size() % sizeof(ResourceId) != 0) + return header; + std::vector out(header.size()); + const size_t count = header.size() / sizeof(ResourceId); + for(size_t i = 0; i < count; ++i) { + ResourceId raw = 0; + std::memcpy(&raw, header.data() + i * sizeof(ResourceId), sizeof(ResourceId)); + ResourceId mapped = mapModelId(raw); + std::memcpy(out.data() + i * sizeof(ResourceId), &mapped, sizeof(ResourceId)); + } + return out; + } + + if(type == EnumAssets::Model) { + try { + TOS::ByteBuffer buffer(header.size(), header.data()); + auto reader = buffer.reader(); + + uint16_t modelCount = reader.readUInt16(); + std::vector models; + models.reserve(modelCount); + for(uint16_t i = 0; i < modelCount; ++i) { + ResourceId id = reader.readUInt32(); + models.push_back(mapModelId(id)); + } + + uint16_t texCount = reader.readUInt16(); + std::vector> pipelines; + pipelines.reserve(texCount); + for(uint16_t i = 0; i < texCount; ++i) { + uint32_t size32 = reader.readUInt32(); + TOS::ByteBuffer pipe; + reader.readBuffer(pipe); + if(pipe.size() != size32) { + warn("Pipeline size mismatch"); + } + std::vector code(pipe.begin(), pipe.end()); + auto result = remapTexturePipelineIds(code, [&](uint32_t id) { + return mapTextureId(static_cast(id)); + }); + if(!result.Ok) { + warn(result.Error); + } + pipelines.emplace_back(std::move(code)); + } + + TOS::ByteBuffer::Writer wr; + wr << uint16_t(models.size()); + for(ResourceId id : models) + wr << id; + wr << uint16_t(pipelines.size()); + for(const auto& pipe : pipelines) { + wr << uint32_t(pipe.size()); + TOS::ByteBuffer pipeBuff(pipe.begin(), pipe.end()); + wr << pipeBuff; + } + + TOS::ByteBuffer out = wr.complite(); + return std::vector(out.begin(), out.end()); + } catch(const std::exception&) { + warn("Failed to rebind model header"); + return header; + } + } + + return header; +} + +} // namespace LV::Client::AssetsHeaderCodec diff --git a/Src/Client/AssetsHeaderCodec.hpp b/Src/Client/AssetsHeaderCodec.hpp new file mode 100644 index 0000000..bcaa738 --- /dev/null +++ b/Src/Client/AssetsHeaderCodec.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include +#include +#include "Common/Abstract.hpp" + +namespace LV::Client::AssetsHeaderCodec { + +struct ParsedHeader { + EnumAssets Type{}; + std::vector ModelDeps; + std::vector TextureDeps; + std::vector> TexturePipelines; +}; + +using MapIdFn = std::function; +using WarnFn = std::function; + +std::optional parseHeader(EnumAssets type, const std::vector& header); + +std::vector rebindHeader(EnumAssets type, const std::vector& header, + const MapIdFn& mapModelId, const MapIdFn& mapTextureId, const WarnFn& warn); + +} // namespace LV::Client::AssetsHeaderCodec diff --git a/Src/Client/AssetsManager.cpp b/Src/Client/AssetsManager.cpp index f1e2cd3..68dce27 100644 --- a/Src/Client/AssetsManager.cpp +++ b/Src/Client/AssetsManager.cpp @@ -1,11 +1,7 @@ #include "AssetsManager.hpp" #include #include -#include -#include #include -#include -#include "Common/Net.hpp" #include "Common/TexturePipelineProgram.hpp" namespace LV::Client { @@ -73,425 +69,217 @@ static std::u8string readOptionalMeta(const fs::path& path) { return readFileBytes(metaPath); } -static std::vector collectTexturePipelineIds(const std::vector& code); - -struct ParsedModelHeader { - std::vector ModelDeps; - std::vector> TexturePipelines; - std::vector TextureDeps; -}; - -std::optional> parseNodestateHeaderBytes(const std::vector& header) { - if(header.empty() || header.size() % sizeof(AssetsManager::AssetId) != 0) - return std::nullopt; - - const size_t count = header.size() / sizeof(AssetsManager::AssetId); - std::vector deps; - deps.resize(count); - for(size_t i = 0; i < count; ++i) { - AssetsManager::AssetId raw = 0; - std::memcpy(&raw, header.data() + i * sizeof(AssetsManager::AssetId), sizeof(AssetsManager::AssetId)); - deps[i] = raw; - } - return deps; -} - -std::optional parseModelHeaderBytes(const std::vector& header) { - if(header.empty()) - return std::nullopt; - - ParsedModelHeader result; - try { - TOS::ByteBuffer buffer(header.size(), header.data()); - auto reader = buffer.reader(); - - uint16_t modelCount = reader.readUInt16(); - result.ModelDeps.reserve(modelCount); - for(uint16_t i = 0; i < modelCount; ++i) - result.ModelDeps.push_back(reader.readUInt32()); - - uint16_t texCount = reader.readUInt16(); - result.TexturePipelines.reserve(texCount); - for(uint16_t i = 0; i < texCount; ++i) { - uint32_t size32 = reader.readUInt32(); - TOS::ByteBuffer pipe; - reader.readBuffer(pipe); - if(pipe.size() != size32) - return std::nullopt; - result.TexturePipelines.emplace_back(pipe.begin(), pipe.end()); - } - - std::unordered_set seen; - for(const auto& pipe : result.TexturePipelines) { - for(uint32_t id : collectTexturePipelineIds(pipe)) { - if(seen.insert(id).second) - result.TextureDeps.push_back(id); - } - } - } catch(const std::exception&) { - return std::nullopt; - } - - return result; -} - -struct PipelineRemapResult { - bool Ok = true; - std::string Error; -}; - -PipelineRemapResult remapTexturePipelineIds(std::vector& code, - const std::function& mapId) -{ - struct Range { - size_t Start = 0; - size_t End = 0; - }; - - enum class SrcKind : uint8_t { TexId = 0, Sub = 1 }; - enum class Op : uint8_t { - End = 0, - Base_Tex = 1, - Base_Fill = 2, - Base_Anim = 3, - Resize = 10, - Transform = 11, - Opacity = 12, - NoAlpha = 13, - MakeAlpha = 14, - Invert = 15, - Brighten = 16, - Contrast = 17, - Multiply = 18, - Screen = 19, - Colorize = 20, - Anim = 21, - Overlay = 30, - Mask = 31, - LowPart = 32, - Combine = 40 - }; - - struct SrcMeta { - SrcKind Kind = SrcKind::TexId; - uint32_t TexId = 0; - uint32_t Off = 0; - uint32_t Len = 0; - size_t TexIdOffset = 0; - }; - - const size_t size = code.size(); - std::vector visited; - - auto read8 = [&](size_t& ip, uint8_t& out) -> bool { - if(ip >= size) - return false; - out = code[ip++]; - return true; - }; - - auto read16 = [&](size_t& ip, uint16_t& out) -> bool { - if(ip + 1 >= size) - return false; - out = uint16_t(code[ip]) | (uint16_t(code[ip + 1]) << 8); - ip += 2; - return true; - }; - - auto read24 = [&](size_t& ip, uint32_t& out) -> bool { - if(ip + 2 >= size) - return false; - out = uint32_t(code[ip]) | - (uint32_t(code[ip + 1]) << 8) | - (uint32_t(code[ip + 2]) << 16); - ip += 3; - return true; - }; - - auto read32 = [&](size_t& ip, uint32_t& out) -> bool { - if(ip + 3 >= size) - return false; - out = uint32_t(code[ip]) | - (uint32_t(code[ip + 1]) << 8) | - (uint32_t(code[ip + 2]) << 16) | - (uint32_t(code[ip + 3]) << 24); - ip += 4; - return true; - }; - - auto readSrc = [&](size_t& ip, SrcMeta& out) -> bool { - uint8_t kind = 0; - if(!read8(ip, kind)) - return false; - out.Kind = static_cast(kind); - if(out.Kind == SrcKind::TexId) { - out.TexIdOffset = ip; - return read24(ip, out.TexId); - } - if(out.Kind == SrcKind::Sub) { - return read24(ip, out.Off) && read24(ip, out.Len); - } - return false; - }; - - auto patchTexId = [&](const SrcMeta& src) -> bool { - if(src.Kind != SrcKind::TexId) - return true; - uint32_t newId = mapId(src.TexId); - if(newId >= (1u << 24)) - return false; - if(src.TexIdOffset + 2 >= code.size()) - return false; - code[src.TexIdOffset + 0] = uint8_t(newId & 0xFFu); - code[src.TexIdOffset + 1] = uint8_t((newId >> 8) & 0xFFu); - code[src.TexIdOffset + 2] = uint8_t((newId >> 16) & 0xFFu); - return true; - }; - - std::move_only_function scan; - scan = [&](size_t start, size_t end) -> bool { - if(start >= end || end > size) - return true; - for(const auto& r : visited) { - if(r.Start == start && r.End == end) - return true; - } - visited.push_back(Range{start, end}); - - size_t ip = start; - while(ip < end) { - uint8_t opByte = 0; - if(!read8(ip, opByte)) - return false; - Op op = static_cast(opByte); - switch(op) { - case Op::End: - return true; - - case Op::Base_Tex: { - SrcMeta src{}; - if(!readSrc(ip, src)) - return false; - if(!patchTexId(src)) - return false; - if(src.Kind == SrcKind::Sub) { - size_t subStart = src.Off; - size_t subEnd = subStart + src.Len; - if(!scan(subStart, subEnd)) - return false; - } - } break; - - case Op::Base_Fill: { - uint16_t tmp16 = 0; - uint32_t tmp32 = 0; - if(!read16(ip, tmp16)) - return false; - if(!read16(ip, tmp16)) - return false; - if(!read32(ip, tmp32)) - return false; - } break; - - case Op::Base_Anim: { - SrcMeta src{}; - if(!readSrc(ip, src)) - return false; - if(!patchTexId(src)) - return false; - uint16_t tmp16 = 0; - uint8_t tmp8 = 0; - if(!read16(ip, tmp16)) - return false; - if(!read16(ip, tmp16)) - return false; - if(!read16(ip, tmp16)) - return false; - if(!read16(ip, tmp16)) - return false; - if(!read8(ip, tmp8)) - return false; - if(src.Kind == SrcKind::Sub) { - size_t subStart = src.Off; - size_t subEnd = subStart + src.Len; - if(!scan(subStart, subEnd)) - return false; - } - } break; - - case Op::Resize: { - uint16_t tmp16 = 0; - if(!read16(ip, tmp16)) - return false; - if(!read16(ip, tmp16)) - return false; - } break; - - case Op::Transform: - case Op::Opacity: - case Op::Invert: { - uint8_t tmp8 = 0; - if(!read8(ip, tmp8)) - return false; - } break; - - case Op::NoAlpha: - case Op::Brighten: - break; - - case Op::MakeAlpha: - if(ip + 2 >= size) - return false; - ip += 3; - break; - - case Op::Contrast: - if(ip + 1 >= size) - return false; - ip += 2; - break; - - case Op::Multiply: - case Op::Screen: { - uint32_t tmp32 = 0; - if(!read32(ip, tmp32)) - return false; - } break; - - case Op::Colorize: { - uint32_t tmp32 = 0; - uint8_t tmp8 = 0; - if(!read32(ip, tmp32)) - return false; - if(!read8(ip, tmp8)) - return false; - } break; - - case Op::Anim: { - uint16_t tmp16 = 0; - uint8_t tmp8 = 0; - if(!read16(ip, tmp16)) - return false; - if(!read16(ip, tmp16)) - return false; - if(!read16(ip, tmp16)) - return false; - if(!read16(ip, tmp16)) - return false; - if(!read8(ip, tmp8)) - return false; - } break; - - case Op::Overlay: - case Op::Mask: { - SrcMeta src{}; - if(!readSrc(ip, src)) - return false; - if(!patchTexId(src)) - return false; - if(src.Kind == SrcKind::Sub) { - size_t subStart = src.Off; - size_t subEnd = subStart + src.Len; - if(!scan(subStart, subEnd)) - return false; - } - } break; - - case Op::LowPart: { - uint8_t tmp8 = 0; - if(!read8(ip, tmp8)) - return false; - SrcMeta src{}; - if(!readSrc(ip, src)) - return false; - if(!patchTexId(src)) - return false; - if(src.Kind == SrcKind::Sub) { - size_t subStart = src.Off; - size_t subEnd = subStart + src.Len; - if(!scan(subStart, subEnd)) - return false; - } - } break; - - case Op::Combine: { - uint16_t tmp16 = 0; - if(!read16(ip, tmp16)) - return false; - if(!read16(ip, tmp16)) - return false; - uint16_t count = 0; - if(!read16(ip, count)) - return false; - for(uint16_t i = 0; i < count; ++i) { - if(!read16(ip, tmp16)) - return false; - if(!read16(ip, tmp16)) - return false; - SrcMeta src{}; - if(!readSrc(ip, src)) - return false; - if(!patchTexId(src)) - return false; - if(src.Kind == SrcKind::Sub) { - size_t subStart = src.Off; - size_t subEnd = subStart + src.Len; - if(!scan(subStart, subEnd)) - return false; - } - } - } break; - - default: - return false; - } - } - - return true; - }; - - if(!scan(0, size)) - return {false, "Invalid texture pipeline bytecode"}; - - return {}; -} - -static std::vector collectTexturePipelineIds(const std::vector& code) { - std::vector out; - std::unordered_set seen; - - auto addId = [&](uint32_t id) { - if(seen.insert(id).second) - out.push_back(id); - }; - - std::vector copy = code; - auto result = remapTexturePipelineIds(copy, [&](uint32_t id) { - addId(id); - return id; - }); - - if(!result.Ok) - return {}; - - return out; -} - } // namespace - AssetsManager::AssetsManager(asio::io_context& ioc, const fs::path& cachePath, size_t maxCacheDirectorySize, size_t maxLifeTime) : Cache(AssetsCacheManager::Create(ioc, cachePath, maxCacheDirectorySize, maxLifeTime)) { for(size_t i = 0; i < static_cast(AssetType::MAX_ENUM); ++i) - NextLocalId[i] = 1; + Types[i].NextLocalId = 1; + initSources(); +} + +void AssetsManager::initSources() { + using SourceResult = AssetsManager::SourceResult; + using SourceStatus = AssetsManager::SourceStatus; + using SourceReady = AssetsManager::SourceReady; + using ResourceKey = AssetsManager::ResourceKey; + using PackResource = AssetsManager::PackResource; + + class PackSource final : public IResourceSource { + public: + explicit PackSource(AssetsManager* manager) : Manager(manager) {} + + SourceResult tryGet(const ResourceKey& key) override { + std::optional pack = Manager->findPackResource(key.Type, key.Domain, key.Key); + if(pack && pack->Hash == key.Hash) + return {SourceStatus::Hit, pack->Res, 0}; + return {SourceStatus::Miss, std::nullopt, 0}; + } + + void collectReady(std::vector&) override {} + + bool isAsync() const override { + return false; + } + + void startPending(std::vector) override {} + + private: + AssetsManager* Manager = nullptr; + }; + + class MemorySource final : public IResourceSource { + public: + explicit MemorySource(AssetsManager* manager) : Manager(manager) {} + + SourceResult tryGet(const ResourceKey& key) override { + auto iter = Manager->MemoryResourcesByHash.find(key.Hash); + if(iter == Manager->MemoryResourcesByHash.end()) + return {SourceStatus::Miss, std::nullopt, 0}; + return {SourceStatus::Hit, iter->second, 0}; + } + + void collectReady(std::vector&) override {} + + bool isAsync() const override { + return false; + } + + void startPending(std::vector) override {} + + private: + AssetsManager* Manager = nullptr; + }; + + class CacheSource final : public IResourceSource { + public: + CacheSource(AssetsManager* manager, size_t sourceIndex) + : Manager(manager), SourceIndex(sourceIndex) {} + + SourceResult tryGet(const ResourceKey&) override { + return {SourceStatus::Pending, std::nullopt, SourceIndex}; + } + + void collectReady(std::vector& out) override { + std::vector>> cached = Manager->Cache->pullReads(); + out.reserve(out.size() + cached.size()); + for(auto& [hash, res] : cached) + out.push_back(SourceReady{hash, res, SourceIndex}); + } + + bool isAsync() const override { + return true; + } + + void startPending(std::vector hashes) override { + if(!hashes.empty()) + Manager->Cache->pushReads(std::move(hashes)); + } + + private: + AssetsManager* Manager = nullptr; + size_t SourceIndex = 0; + }; + + Sources.clear(); + PackSourceIndex = Sources.size(); + Sources.push_back(SourceEntry{std::make_unique(this), 0}); + MemorySourceIndex = Sources.size(); + Sources.push_back(SourceEntry{std::make_unique(this), 0}); + CacheSourceIndex = Sources.size(); + Sources.push_back(SourceEntry{std::make_unique(this, CacheSourceIndex), 0}); +} + +void AssetsManager::collectReadyFromSources() { + std::vector ready; + for(auto& entry : Sources) + entry.Source->collectReady(ready); + + for(SourceReady& item : ready) { + auto iter = PendingReadsByHash.find(item.Hash); + if(iter == PendingReadsByHash.end()) + continue; + if(item.Value) + registerSourceHit(item.Hash, item.SourceIndex); + for(ResourceKey& key : iter->second) { + if(item.SourceIndex == CacheSourceIndex) { + if(item.Value) { + LOG.debug() << "Cache hit type=" << assetTypeName(key.Type) + << " id=" << key.Id + << " key=" << key.Domain << ':' << key.Key + << " hash=" << int(item.Hash[0]) << '.' + << int(item.Hash[1]) << '.' + << int(item.Hash[2]) << '.' + << int(item.Hash[3]) + << " size=" << item.Value->size(); + } else { + LOG.debug() << "Cache miss type=" << assetTypeName(key.Type) + << " id=" << key.Id + << " key=" << key.Domain << ':' << key.Key + << " hash=" << int(item.Hash[0]) << '.' + << int(item.Hash[1]) << '.' + << int(item.Hash[2]) << '.' + << int(item.Hash[3]); + } + } + ReadyReads.emplace_back(std::move(key), item.Value); + } + PendingReadsByHash.erase(iter); + } +} + +AssetsManager::SourceResult AssetsManager::querySources(const ResourceKey& key) { + auto cacheIter = SourceCacheByHash.find(key.Hash); + if(cacheIter != SourceCacheByHash.end()) { + const size_t cachedIndex = cacheIter->second.SourceIndex; + if(cachedIndex < Sources.size() + && cacheIter->second.Generation == Sources[cachedIndex].Generation) + { + SourceResult cached = Sources[cachedIndex].Source->tryGet(key); + cached.SourceIndex = cachedIndex; + if(cached.Status != SourceStatus::Miss) + return cached; + } + SourceCacheByHash.erase(cacheIter); + } + + SourceResult pending; + pending.Status = SourceStatus::Miss; + for(size_t i = 0; i < Sources.size(); ++i) { + SourceResult res = Sources[i].Source->tryGet(key); + res.SourceIndex = i; + if(res.Status == SourceStatus::Hit) { + registerSourceHit(key.Hash, i); + return res; + } + if(res.Status == SourceStatus::Pending && pending.Status == SourceStatus::Miss) + pending = res; + } + + return pending; +} + +void AssetsManager::registerSourceHit(const Hash_t& hash, size_t sourceIndex) { + if(sourceIndex >= Sources.size()) + return; + if(Sources[sourceIndex].Source->isAsync()) + return; + SourceCacheByHash[hash] = SourceCacheEntry{ + .SourceIndex = sourceIndex, + .Generation = Sources[sourceIndex].Generation + }; +} + +void AssetsManager::invalidateSourceCache(size_t sourceIndex) { + if(sourceIndex >= Sources.size()) + return; + Sources[sourceIndex].Generation++; + for(auto iter = SourceCacheByHash.begin(); iter != SourceCacheByHash.end(); ) { + if(iter->second.SourceIndex == sourceIndex) + iter = SourceCacheByHash.erase(iter); + else + ++iter; + } +} + +void AssetsManager::invalidateAllSourceCache() { + for(auto& entry : Sources) + entry.Generation++; + SourceCacheByHash.clear(); +} + +void AssetsManager::tickSources() { + collectReadyFromSources(); } AssetsManager::PackReloadResult AssetsManager::reloadPacks(const PackRegister& reg) { PackReloadResult result; - auto oldPacks = PackResources; - for(auto& table : PackResources) - table.clear(); + std::array(AssetType::MAX_ENUM)> oldPacks; + for(size_t type = 0; type < static_cast(AssetType::MAX_ENUM); ++type) { + oldPacks[type] = Types[type].PackResources; + Types[type].PackResources.clear(); + } for(const fs::path& instance : reg.Packs) { try { @@ -522,7 +310,7 @@ AssetsManager::PackReloadResult AssetsManager::reloadPacks(const PackRegister& r if(!fs::exists(assetPath) || !fs::is_directory(assetPath)) continue; - auto& typeTable = PackResources[type][domain]; + auto& typeTable = Types[type].PackResources[domain]; for(auto fbegin = fs::recursive_directory_iterator(assetPath), fend = fs::recursive_directory_iterator(); fbegin != fend; ++fbegin) { @@ -532,7 +320,7 @@ AssetsManager::PackReloadResult AssetsManager::reloadPacks(const PackRegister& r if(assetType == AssetType::Texture && file.extension() == ".meta") continue; - std::string key = fs::relative(file, assetPath).string(); + std::string key = fs::relative(file, assetPath).generic_string(); if(typeTable.contains(key)) continue; @@ -630,7 +418,7 @@ AssetsManager::PackReloadResult AssetsManager::reloadPacks(const PackRegister& r } for(size_t type = 0; type < static_cast(AssetType::MAX_ENUM); ++type) { - for(const auto& [domain, keyTable] : PackResources[type]) { + for(const auto& [domain, keyTable] : Types[type].PackResources) { for(const auto& [key, res] : keyTable) { bool changed = true; auto oldDomain = oldPacks[type].find(domain); @@ -647,9 +435,9 @@ AssetsManager::PackReloadResult AssetsManager::reloadPacks(const PackRegister& r for(const auto& [domain, keyTable] : oldPacks[type]) { for(const auto& [key, res] : keyTable) { - auto newDomain = PackResources[type].find(domain); + auto newDomain = Types[type].PackResources.find(domain); bool lost = true; - if(newDomain != PackResources[type].end()) { + if(newDomain != Types[type].PackResources.end()) { if(newDomain->second.contains(key)) lost = false; } @@ -659,6 +447,7 @@ AssetsManager::PackReloadResult AssetsManager::reloadPacks(const PackRegister& r } } + invalidateAllSourceCache(); return result; } @@ -667,7 +456,7 @@ AssetsManager::BindResult AssetsManager::bindServerResource(AssetType type, Asse { BindResult result; AssetId localFromDK = getOrCreateLocalId(type, domain, key); - auto& map = ServerToLocal[static_cast(type)]; + auto& map = Types[static_cast(type)].ServerToLocal; AssetId localFromServer = 0; if(serverId < map.size()) localFromServer = map[serverId]; @@ -680,7 +469,7 @@ AssetsManager::BindResult AssetsManager::bindServerResource(AssetType type, Asse map.resize(serverId + 1, 0); map[serverId] = localId; - auto& infoList = BindInfos[static_cast(type)]; + auto& infoList = Types[static_cast(type)].BindInfos; if(localId >= infoList.size()) infoList.resize(localId + 1); @@ -703,7 +492,7 @@ AssetsManager::BindResult AssetsManager::bindServerResource(AssetType type, Asse } std::optional AssetsManager::unbindServerResource(AssetType type, AssetId serverId) { - auto& map = ServerToLocal[static_cast(type)]; + auto& map = Types[static_cast(type)].ServerToLocal; if(serverId >= map.size()) return std::nullopt; AssetId localId = map[serverId]; @@ -714,15 +503,15 @@ std::optional AssetsManager::unbindServerResource(AssetT } void AssetsManager::clearServerBindings() { - for(auto& table : ServerToLocal) - table.clear(); - for(auto& table : BindInfos) - table.clear(); + for(auto& typeData : Types) { + typeData.ServerToLocal.clear(); + typeData.BindInfos.clear(); + } } const AssetsManager::BindInfo* AssetsManager::getBind(AssetType type, AssetId localId) const { localId = resolveLocalId(type, localId); - const auto& table = BindInfos[static_cast(type)]; + const auto& table = Types[static_cast(type)].BindInfos; if(localId >= table.size()) return nullptr; if(!table[localId]) @@ -731,9 +520,6 @@ const AssetsManager::BindInfo* AssetsManager::getBind(AssetType type, AssetId lo } std::vector AssetsManager::rebindHeader(AssetType type, const std::vector& header, bool serverIds) { - if(header.empty()) - return {}; - auto mapModelId = [&](AssetId id) -> AssetId { if(serverIds) { auto localId = getLocalIdFromServer(AssetType::Model, id); @@ -758,134 +544,76 @@ std::vector AssetsManager::rebindHeader(AssetType type, const std::vect return resolveLocalIdMutable(AssetType::Texture, id); }; - if(type == AssetType::Nodestate) { - if(header.size() % sizeof(AssetId) != 0) - return header; - std::vector out(header.size()); - const size_t count = header.size() / sizeof(AssetId); - for(size_t i = 0; i < count; ++i) { - AssetId raw = 0; - std::memcpy(&raw, header.data() + i * sizeof(AssetId), sizeof(AssetId)); - AssetId mapped = mapModelId(raw); - std::memcpy(out.data() + i * sizeof(AssetId), &mapped, sizeof(AssetId)); - } - return out; - } + auto warn = [&](const std::string& msg) { + LOG.warn() << msg; + }; - if(type == AssetType::Model) { - try { - TOS::ByteBuffer buffer(header.size(), header.data()); - auto reader = buffer.reader(); - - uint16_t modelCount = reader.readUInt16(); - std::vector models; - models.reserve(modelCount); - for(uint16_t i = 0; i < modelCount; ++i) { - AssetId id = reader.readUInt32(); - models.push_back(mapModelId(id)); - } - - uint16_t texCount = reader.readUInt16(); - std::vector> pipelines; - pipelines.reserve(texCount); - for(uint16_t i = 0; i < texCount; ++i) { - uint32_t size32 = reader.readUInt32(); - TOS::ByteBuffer pipe; - reader.readBuffer(pipe); - if(pipe.size() != size32) { - LOG.warn() << "Несовпадение длины pipeline: " << size32 << " vs " << pipe.size(); - } - std::vector code(pipe.begin(), pipe.end()); - auto result = remapTexturePipelineIds(code, [&](uint32_t id) { - return mapTextureId(static_cast(id)); - }); - if(!result.Ok) { - LOG.warn() << "Ошибка ребинда pipeline: " << result.Error; - } - pipelines.emplace_back(std::move(code)); - } - - TOS::ByteBuffer::Writer wr; - wr << uint16_t(models.size()); - for(AssetId id : models) - wr << id; - wr << uint16_t(pipelines.size()); - for(const auto& pipe : pipelines) { - wr << uint32_t(pipe.size()); - TOS::ByteBuffer pipeBuff(pipe.begin(), pipe.end()); - wr << pipeBuff; - } - - TOS::ByteBuffer out = wr.complite(); - return std::vector(out.begin(), out.end()); - } catch(const std::exception& exc) { - LOG.warn() << "Ошибка ребинда заголовка модели: " << exc.what(); - return header; - } - } - - return header; + return AssetsHeaderCodec::rebindHeader(type, header, mapModelId, mapTextureId, warn); } std::optional AssetsManager::parseHeader(AssetType type, const std::vector& header) { - if(header.empty()) - return std::nullopt; + return AssetsHeaderCodec::parseHeader(type, header); +} - ParsedHeader result; - result.Type = type; +void AssetsManager::pushResources(std::vector resources) { + for(const Resource& res : resources) { + Hash_t hash = res.hash(); + MemoryResourcesByHash[hash] = res; + SourceCacheByHash.erase(hash); + registerSourceHit(hash, MemorySourceIndex); - if(type == AssetType::Nodestate) { - auto deps = parseNodestateHeaderBytes(header); - if(!deps) - return std::nullopt; - result.ModelDeps = std::move(*deps); - return result; + auto iter = PendingReadsByHash.find(hash); + if(iter != PendingReadsByHash.end()) { + for(ResourceKey& key : iter->second) + ReadyReads.emplace_back(std::move(key), res); + PendingReadsByHash.erase(iter); + } } - if(type == AssetType::Model) { - auto parsed = parseModelHeaderBytes(header); - if(!parsed) - return std::nullopt; - result.ModelDeps = std::move(parsed->ModelDeps); - result.TexturePipelines = std::move(parsed->TexturePipelines); - result.TextureDeps = std::move(parsed->TextureDeps); - return result; - } - - return std::nullopt; + Cache->pushResources(std::move(resources)); } void AssetsManager::pushReads(std::vector reads) { - std::vector forCache; - forCache.reserve(reads.size()); + std::unordered_map> pendingBySource; for(ResourceKey& key : reads) { - std::optional pack = findPackResource(key.Type, key.Domain, key.Key); - if(pack && pack->Hash == key.Hash) { - LOG.debug() << "Pack hit type=" << assetTypeName(key.Type) - << " id=" << key.Id - << " key=" << key.Domain << ':' << key.Key - << " hash=" << int(key.Hash[0]) << '.' - << int(key.Hash[1]) << '.' - << int(key.Hash[2]) << '.' - << int(key.Hash[3]) - << " size=" << pack->Res.size(); - ReadyReads.emplace_back(std::move(key), pack->Res); + SourceResult res = querySources(key); + if(res.Status == SourceStatus::Hit) { + if(res.SourceIndex == PackSourceIndex && res.Value) { + LOG.debug() << "Pack hit type=" << assetTypeName(key.Type) + << " id=" << key.Id + << " key=" << key.Domain << ':' << key.Key + << " hash=" << int(key.Hash[0]) << '.' + << int(key.Hash[1]) << '.' + << int(key.Hash[2]) << '.' + << int(key.Hash[3]) + << " size=" << res.Value->size(); + } + ReadyReads.emplace_back(std::move(key), res.Value); continue; } - auto& list = PendingReadsByHash[key.Hash]; - bool isFirst = list.empty(); - list.push_back(std::move(key)); - if(isFirst) - forCache.push_back(list.front().Hash); + if(res.Status == SourceStatus::Pending) { + auto& list = PendingReadsByHash[key.Hash]; + bool isFirst = list.empty(); + list.push_back(std::move(key)); + if(isFirst) + pendingBySource[res.SourceIndex].push_back(list.front().Hash); + continue; + } + + ReadyReads.emplace_back(std::move(key), std::nullopt); } - if(!forCache.empty()) - Cache->pushReads(std::move(forCache)); + for(auto& [sourceIndex, hashes] : pendingBySource) { + if(sourceIndex < Sources.size()) + Sources[sourceIndex].Source->startPending(std::move(hashes)); + } } std::vector>> AssetsManager::pullReads() { + tickSources(); + std::vector>> out; out.reserve(ReadyReads.size()); @@ -893,40 +621,11 @@ std::vector>> Asse out.emplace_back(std::move(entry)); ReadyReads.clear(); - std::vector>> cached = Cache->pullReads(); - for(auto& [hash, res] : cached) { - auto iter = PendingReadsByHash.find(hash); - if(iter == PendingReadsByHash.end()) - continue; - for(ResourceKey& key : iter->second) { - if(res) { - LOG.debug() << "Cache hit type=" << assetTypeName(key.Type) - << " id=" << key.Id - << " key=" << key.Domain << ':' << key.Key - << " hash=" << int(hash[0]) << '.' - << int(hash[1]) << '.' - << int(hash[2]) << '.' - << int(hash[3]) - << " size=" << res->size(); - } else { - LOG.debug() << "Cache miss type=" << assetTypeName(key.Type) - << " id=" << key.Id - << " key=" << key.Domain << ':' << key.Key - << " hash=" << int(hash[0]) << '.' - << int(hash[1]) << '.' - << int(hash[2]) << '.' - << int(hash[3]); - } - out.emplace_back(std::move(key), res); - } - PendingReadsByHash.erase(iter); - } - return out; } AssetsManager::AssetId AssetsManager::getOrCreateLocalId(AssetType type, std::string_view domain, std::string_view key) { - auto& table = DKToLocal[static_cast(type)]; + auto& table = Types[static_cast(type)].DKToLocal; auto iterDomain = table.find(domain); if(iterDomain == table.end()) { iterDomain = table.emplace( @@ -945,7 +644,7 @@ AssetsManager::AssetId AssetsManager::getOrCreateLocalId(AssetType type, std::st AssetId id = allocateLocalId(type); keyTable.emplace(std::string(key), id); - auto& dk = LocalToDK[static_cast(type)]; + auto& dk = Types[static_cast(type)].LocalToDK; if(id >= dk.size()) dk.resize(id + 1); dk[id] = DomainKey{std::string(domain), std::string(key), true}; @@ -954,7 +653,7 @@ AssetsManager::AssetId AssetsManager::getOrCreateLocalId(AssetType type, std::st } std::optional AssetsManager::getLocalIdFromServer(AssetType type, AssetId serverId) const { - const auto& map = ServerToLocal[static_cast(type)]; + const auto& map = Types[static_cast(type)].ServerToLocal; if(serverId >= map.size()) return std::nullopt; AssetId local = map[serverId]; @@ -966,7 +665,7 @@ std::optional AssetsManager::getLocalIdFromServer(AssetT AssetsManager::AssetId AssetsManager::resolveLocalId(AssetType type, AssetId localId) const { if(localId == 0) return 0; - const auto& parents = LocalParent[static_cast(type)]; + const auto& parents = Types[static_cast(type)].LocalParent; if(localId >= parents.size()) return localId; AssetId cur = localId; @@ -976,15 +675,15 @@ AssetsManager::AssetId AssetsManager::resolveLocalId(AssetType type, AssetId loc } AssetsManager::AssetId AssetsManager::allocateLocalId(AssetType type) { - auto& next = NextLocalId[static_cast(type)]; + auto& next = Types[static_cast(type)].NextLocalId; AssetId id = next++; - auto& parents = LocalParent[static_cast(type)]; + auto& parents = Types[static_cast(type)].LocalParent; if(id >= parents.size()) parents.resize(id + 1, 0); parents[id] = id; - auto& dk = LocalToDK[static_cast(type)]; + auto& dk = Types[static_cast(type)].LocalToDK; if(id >= dk.size()) dk.resize(id + 1); @@ -994,7 +693,7 @@ AssetsManager::AssetId AssetsManager::allocateLocalId(AssetType type) { AssetsManager::AssetId AssetsManager::resolveLocalIdMutable(AssetType type, AssetId localId) { if(localId == 0) return 0; - auto& parents = LocalParent[static_cast(type)]; + auto& parents = Types[static_cast(type)].LocalParent; if(localId >= parents.size()) return localId; AssetId root = localId; @@ -1017,7 +716,7 @@ void AssetsManager::unionLocalIds(AssetType type, AssetId fromId, AssetId toId, if(fromRoot == 0 || toRoot == 0 || fromRoot == toRoot) return; - auto& parents = LocalParent[static_cast(type)]; + auto& parents = Types[static_cast(type)].LocalParent; if(fromRoot >= parents.size() || toRoot >= parents.size()) return; @@ -1025,7 +724,7 @@ void AssetsManager::unionLocalIds(AssetType type, AssetId fromId, AssetId toId, if(reboundFrom) *reboundFrom = fromRoot; - auto& dk = LocalToDK[static_cast(type)]; + auto& dk = Types[static_cast(type)].LocalToDK; if(fromRoot < dk.size()) { const DomainKey& fromDK = dk[fromRoot]; if(fromDK.Known) { @@ -1034,7 +733,7 @@ void AssetsManager::unionLocalIds(AssetType type, AssetId fromId, AssetId toId, DomainKey& toDK = dk[toRoot]; if(!toDK.Known) { toDK = fromDK; - DKToLocal[static_cast(type)][toDK.Domain][toDK.Key] = toRoot; + Types[static_cast(type)].DKToLocal[toDK.Domain][toDK.Key] = toRoot; } else if(toDK.Domain != fromDK.Domain || toDK.Key != fromDK.Key) { LOG.warn() << "Конфликт домен/ключ при ребинде: " << fromDK.Domain << ':' << fromDK.Key << " vs " @@ -1043,7 +742,7 @@ void AssetsManager::unionLocalIds(AssetType type, AssetId fromId, AssetId toId, } } - auto& binds = BindInfos[static_cast(type)]; + auto& binds = Types[static_cast(type)].BindInfos; if(fromRoot < binds.size()) { if(toRoot >= binds.size()) binds.resize(toRoot + 1); @@ -1055,7 +754,7 @@ void AssetsManager::unionLocalIds(AssetType type, AssetId fromId, AssetId toId, std::optional AssetsManager::findPackResource(AssetType type, std::string_view domain, std::string_view key) const { - const auto& typeTable = PackResources[static_cast(type)]; + const auto& typeTable = Types[static_cast(type)].PackResources; auto iterDomain = typeTable.find(domain); if(iterDomain == typeTable.end()) return std::nullopt; diff --git a/Src/Client/AssetsManager.hpp b/Src/Client/AssetsManager.hpp index af8c8cd..be56591 100644 --- a/Src/Client/AssetsManager.hpp +++ b/Src/Client/AssetsManager.hpp @@ -12,6 +12,7 @@ #include #include #include "Client/AssetsCacheManager.hpp" +#include "Client/AssetsHeaderCodec.hpp" #include "Common/Abstract.hpp" #include "TOSLib.hpp" @@ -25,88 +26,131 @@ public: using AssetType = EnumAssets; using AssetId = ResourceId; + // Ключ запроса ресурса (идентификация + хеш для поиска источника). struct ResourceKey { + // Хеш ресурса, используемый для поиска в источниках и кэше. Hash_t Hash{}; + // Тип ресурса (модель, текстура и т.д.). AssetType Type{}; + // Домен ресурса. std::string Domain; + // Ключ ресурса внутри домена. std::string Key; + // Идентификатор ресурса на стороне клиента/локальный. AssetId Id = 0; }; + // Информация о биндинге серверного ресурса на локальный id. struct BindInfo { + // Тип ресурса. AssetType Type{}; + // Локальный идентификатор. AssetId LocalId = 0; + // Домен ресурса. std::string Domain; + // Ключ ресурса. std::string Key; + // Хеш ресурса. Hash_t Hash{}; + // Бинарный заголовок с зависимостями. std::vector Header; }; + // Результат биндинга ресурса сервера. struct BindResult { + // Итоговый локальный идентификатор. AssetId LocalId = 0; + // Признак изменения бинда (хеш/заголовок). bool Changed = false; + // Признак новой привязки. bool NewBinding = false; + // Идентификатор, от которого произошёл ребинд (если был). std::optional ReboundFrom; }; + // Регистрация набора ресурспаков. struct PackRegister { + // Пути до паков (директории/архивы). std::vector Packs; }; + // Ресурс, собранный из пака. struct PackResource { + // Тип ресурса. AssetType Type{}; + // Локальный идентификатор. AssetId LocalId = 0; + // Домен ресурса. std::string Domain; + // Ключ ресурса. std::string Key; + // Тело ресурса. Resource Res; + // Хеш ресурса. Hash_t Hash{}; + // Заголовок ресурса (например, зависимости). std::u8string Header; }; + // Результат пересканирования паков. struct PackReloadResult { + // Добавленные/изменённые ресурсы по типам. std::array, static_cast(AssetType::MAX_ENUM)> ChangeOrAdd; + // Потерянные ресурсы по типам. std::array, static_cast(AssetType::MAX_ENUM)> Lost; }; - struct ParsedHeader { - AssetType Type{}; - std::vector ModelDeps; - std::vector TextureDeps; - std::vector> TexturePipelines; - }; + using ParsedHeader = AssetsHeaderCodec::ParsedHeader; + // Фабрика с настройкой лимитов кэша. static Ptr Create(asio::io_context& ioc, const fs::path& cachePath, size_t maxCacheDirectorySize = 8 * 1024 * 1024 * 1024ULL, size_t maxLifeTime = 7 * 24 * 60 * 60) { return Ptr(new AssetsManager(ioc, cachePath, maxCacheDirectorySize, maxLifeTime)); } + // Пересканировать ресурспаки и вернуть изменившиеся/утраченные ресурсы. PackReloadResult reloadPacks(const PackRegister& reg); + // Связать серверный ресурс с локальным id и записать метаданные. BindResult bindServerResource(AssetType type, AssetId serverId, std::string domain, std::string key, const Hash_t& hash, std::vector header); + // Отвязать серверный id и вернуть актуальный локальный id (если был). std::optional unbindServerResource(AssetType type, AssetId serverId); + // Сбросить все серверные бинды. void clearServerBindings(); + // Получить данные бинда по локальному id. const BindInfo* getBind(AssetType type, AssetId localId) const; + // Перебиндить хедер, заменив id зависимостей. std::vector rebindHeader(AssetType type, const std::vector& header, bool serverIds = true); + // Распарсить хедер ресурса. static std::optional parseHeader(AssetType type, const std::vector& header); - void pushResources(std::vector resources) { - Cache->pushResources(std::move(resources)); - } + // Протолкнуть новые ресурсы в память и кэш. + void pushResources(std::vector resources); + // Поставить запросы чтения ресурсов. void pushReads(std::vector reads); + // Получить готовые результаты чтения. std::vector>> pullReads(); + // Продвинуть асинхронные источники (кэш). + void tickSources(); + // Получить или создать локальный id по домену/ключу. AssetId getOrCreateLocalId(AssetType type, std::string_view domain, std::string_view key); + // Получить локальный id по серверному id (если есть). std::optional getLocalIdFromServer(AssetType type, AssetId serverId) const; private: + // Связка домен/ключ для локального id. struct DomainKey { + // Домен ресурса. std::string Domain; + // Ключ ресурса. std::string Key; + // Признак валидности записи. bool Known = false; }; @@ -122,28 +166,127 @@ private: detail::TSVHash, detail::TSVEq>; + struct PerType { + // Таблица домен/ключ -> локальный id. + IdTable DKToLocal; + // Таблица локальный id -> домен/ключ. + std::vector LocalToDK; + // Union-Find родительские ссылки для ребиндов. + std::vector LocalParent; + // Таблица серверный id -> локальный id. + std::vector ServerToLocal; + // Бинды с сервером по локальному id. + std::vector> BindInfos; + // Ресурсы, собранные из паков. + PackTable PackResources; + // Следующий локальный id. + AssetId NextLocalId = 1; + }; + + enum class SourceStatus { + Hit, + Miss, + Pending + }; + + struct SourceResult { + // Статус ответа источника. + SourceStatus Status = SourceStatus::Miss; + // Значение ресурса, если найден. + std::optional Value; + // Индекс источника. + size_t SourceIndex = 0; + }; + + struct SourceReady { + // Хеш готового ресурса. + Hash_t Hash{}; + // Значение ресурса, если найден. + std::optional Value; + // Индекс источника. + size_t SourceIndex = 0; + }; + + class IResourceSource { + public: + virtual ~IResourceSource() = default; + // Попытка получить ресурс синхронно. + virtual SourceResult tryGet(const ResourceKey& key) = 0; + // Забрать готовые результаты асинхронных запросов. + virtual void collectReady(std::vector& out) = 0; + // Признак асинхронности источника. + virtual bool isAsync() const = 0; + // Запустить асинхронные запросы по хешам. + virtual void startPending(std::vector hashes) = 0; + }; + + struct SourceEntry { + // Экземпляр источника. + std::unique_ptr Source; + // Поколение для инвалидирования кэша. + size_t Generation = 0; + }; + + struct SourceCacheEntry { + // Индекс источника, где был найден хеш. + size_t SourceIndex = 0; + // Поколение источника на момент кэширования. + size_t Generation = 0; + }; + + // Конструктор с зависимостью от io_context и кэш-пути. AssetsManager(asio::io_context& ioc, const fs::path& cachePath, size_t maxCacheDirectorySize, size_t maxLifeTime); + // Инициализация списка источников. + void initSources(); + // Забрать готовые результаты из источников. + void collectReadyFromSources(); + // Запросить ресурс в источниках, с учётом кэша. + SourceResult querySources(const ResourceKey& key); + // Запомнить успешный источник для хеша. + void registerSourceHit(const Hash_t& hash, size_t sourceIndex); + // Инвалидировать кэш по конкретному источнику. + void invalidateSourceCache(size_t sourceIndex); + // Инвалидировать весь кэш источников. + void invalidateAllSourceCache(); + + // Выделить новый локальный id. AssetId allocateLocalId(AssetType type); + // Получить корневой локальный id с компрессией пути. AssetId resolveLocalIdMutable(AssetType type, AssetId localId); + // Получить корневой локальный id без мутаций. AssetId resolveLocalId(AssetType type, AssetId localId) const; + // Объединить два локальных id в один. void unionLocalIds(AssetType type, AssetId fromId, AssetId toId, std::optional* reboundFrom); + // Найти ресурс в паке по домену/ключу. std::optional findPackResource(AssetType type, std::string_view domain, std::string_view key) const; + // Логгер подсистемы. Logger LOG = "Client>AssetsManager"; + // Менеджер файлового кэша. AssetsCacheManager::Ptr Cache; - std::array(AssetType::MAX_ENUM)> DKToLocal; - std::array, static_cast(AssetType::MAX_ENUM)> LocalToDK; - std::array, static_cast(AssetType::MAX_ENUM)> LocalParent; - std::array, static_cast(AssetType::MAX_ENUM)> ServerToLocal; - std::array>, static_cast(AssetType::MAX_ENUM)> BindInfos; - std::array(AssetType::MAX_ENUM)> PackResources; - std::array(AssetType::MAX_ENUM)> NextLocalId{}; + // Таблицы данных по каждому типу ресурсов. + std::array(AssetType::MAX_ENUM)> Types; + // Список источников ресурсов. + std::vector Sources; + // Кэш попаданий по хешу. + std::unordered_map SourceCacheByHash; + // Индекс источника паков. + size_t PackSourceIndex = 0; + // Индекс памяти (RAM) как источника. + size_t MemorySourceIndex = 0; + // Индекс файлового кэша. + size_t CacheSourceIndex = 0; + + // Ресурсы в памяти по хешу. + std::unordered_map MemoryResourcesByHash; + // Ожидающие запросы, сгруппированные по хешу. std::unordered_map> PendingReadsByHash; + // Готовые ответы на чтение. std::vector>> ReadyReads; }; diff --git a/Src/Common/AssetsPreloader.cpp b/Src/Common/AssetsPreloader.cpp index 5be489c..2f08593 100644 --- a/Src/Common/AssetsPreloader.cpp +++ b/Src/Common/AssetsPreloader.cpp @@ -126,7 +126,7 @@ AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const Ass if (assetType == AssetType::Texture && file.extension() == ".meta") continue; - std::string key = fs::relative(file, assetPath).string(); + std::string key = fs::relative(file, assetPath).generic_string(); if (firstStage.contains(key)) continue; @@ -197,7 +197,7 @@ AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const Ass fs::path pKeyPath = fs::path(pKeyRaw); if(pKeyPath.extension().empty()) pKeyPath += ".json"; - std::string pKey = pKeyPath.string(); + std::string pKey = pKeyPath.generic_string(); std::optional parent = loadModelProfile(pDomain, pKey, visiting); if(parent) { diff --git a/docs/assets_manager.md b/docs/assets_manager.md new file mode 100644 index 0000000..dbebc75 --- /dev/null +++ b/docs/assets_manager.md @@ -0,0 +1,66 @@ +# AssetsManager + +Документ описывает реализацию `AssetsManager` на стороне клиента. + +## Назначение + +`AssetsManager` объединяет в одном объекте: +- таблицы привязок ресурсов (domain/key -> localId, serverId -> localId); +- загрузку и хранение ресурсов из ресурспаков; +- систему источников данных (packs/memory/cache) с асинхронной выдачей; +- перестройку заголовков ресурсов под локальные идентификаторы. + +## Основные структуры данных + +- `PerType` — набор таблиц на каждый тип ресурса: + - `DKToLocal` и `LocalToDK` — двунаправленное отображение domain/key <-> localId. + - `LocalParent` — union-find для ребиндинга локальных id. + - `ServerToLocal` и `BindInfos` — привязки серверных id и их метаданные. + - `PackResources` — набор ресурсов, собранных из паков. +- `Sources` — список источников ресурсов (pack, memory, cache). +- `SourceCacheByHash` — кэш успешного источника для хеша. +- `PendingReadsByHash` и `ReadyReads` — очередь ожидания и готовые ответы. + +## Источники ресурсов + +Источники реализованы через интерфейс `IResourceSource`: +- pack source (sync) — ищет ресурсы в `PackResources`. +- memory source (sync) — ищет в `MemoryResourcesByHash`. +- cache source (async) — делает чтения через `AssetsCacheManager`. + +Алгоритм поиска: +1) Сначала проверяется `SourceCacheByHash` (если не протух по поколению). +2) Источники опрашиваются по порядку, первый `Hit` возвращается сразу. +3) Если источник вернул `Pending`, запрос попадает в ожидание. +4) `tickSources()` опрашивает асинхронные источники и переводит ответы в `ReadyReads`. + +## Привязка идентификаторов + +- `getOrCreateLocalId()` создаёт локальный id для domain/key. +- `bindServerResource()` связывает serverId с localId и записывает `BindInfo`. +- `unionLocalIds()` объединяет локальные id при конфликте, используя union-find. + +## Ресурспаки + +`reloadPacks()` сканирует директории, собирает ресурсы в `PackResources`, +а затем возвращает список изменений и потерь по типам. + +Важно: ключи ресурсов всегда хранятся с разделителем `/`. +Для нормализации пути используется `fs::path::generic_string()`. + +## Заголовки + +- `rebindHeader()` заменяет id зависимостей в заголовках ресурса. +- `parseHeader()` парсит заголовок без модификаций. + +## Поток данных чтения + +1) `pushReads()` принимает список `ResourceKey` и пытается получить ресурс. +2) `pullReads()` возвращает готовые ответы, включая промахи. +3) `pushResources()` добавляет ресурсы в память и прокидывает их в кэш. + +## Ограничения + +- Класс не предназначен для внешнего многопоточного использования. +- Политика приоритета ресурсов в паке фиксированная: первый найденный ключ побеждает. +- Коллизии хешей не обрабатываются отдельно.