From 5135aa30a7273fdc8412661ca01787c8764dc1d0 Mon Sep 17 00:00:00 2001 From: DrSocalkwe3n Date: Wed, 7 Jan 2026 04:03:17 +0600 Subject: [PATCH] =?UTF-8?q?codex-5.2:=20=D1=82=D0=B5=D1=81=D1=82=20=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=BE=D0=B9=20=D0=B2=D0=B5=D1=80=D1=81=D0=B8=D0=B8?= =?UTF-8?q?=20=D0=BC=D0=B5=D0=BD=D0=B5=D0=B4=D0=B6=D0=B5=D1=80=D0=B0=20?= =?UTF-8?q?=D0=B0=D1=81=D1=81=D0=B5=D1=82=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Src/Client/Abstract.hpp | 21 +- Src/Client/AssetsManager.hpp | 279 +++++++++++++++++++++- Src/Client/ServerSession.cpp | 138 +++++++++-- Src/Client/Vulkan/VulkanRenderSession.cpp | 12 +- Src/Client/Vulkan/VulkanRenderSession.hpp | 230 ++++++------------ Src/Common/Abstract.cpp | 161 +++++++++---- Src/Common/Abstract.hpp | 18 ++ 7 files changed, 607 insertions(+), 252 deletions(-) diff --git a/Src/Client/Abstract.hpp b/Src/Client/Abstract.hpp index 2e9a9c0..ca81474 100644 --- a/Src/Client/Abstract.hpp +++ b/Src/Client/Abstract.hpp @@ -158,6 +158,25 @@ struct DefNode_t { }; +struct AssetEntry { + ResourceId Id = 0; + std::string Domain; + std::string Key; + + HeadlessModel Model; + HeadlessModel::Header ModelHeader; + + HeadlessNodeState Nodestate; + HeadlessNodeState::Header NodestateHeader; + + uint16_t Width = 0; + uint16_t Height = 0; + std::vector Pixels; + ResourceHeader Header; + + std::u8string Data; +}; + /* Интерфейс обработчика сессии с сервером. @@ -172,7 +191,7 @@ public: bool DebugLogPackets = false; // Используемые двоичные ресурсы - // std::unordered_map> Assets; + std::unordered_map> Assets; // Используемые профили контента struct { diff --git a/Src/Client/AssetsManager.hpp b/Src/Client/AssetsManager.hpp index a2e0bb1..f653c91 100644 --- a/Src/Client/AssetsManager.hpp +++ b/Src/Client/AssetsManager.hpp @@ -13,13 +13,17 @@ #include #include #include +#include #include "Client/AssetsCacheManager.hpp" #include "Client/AssetsHeaderCodec.hpp" #include "Common/Abstract.hpp" #include "Common/IdProvider.hpp" #include "Common/AssetsPreloader.hpp" +#include "Common/TexturePipelineProgram.hpp" #include "TOSLib.hpp" +#include "assets.hpp" #include "boost/asio/io_context.hpp" +#include "png++/image.hpp" #include namespace LV::Client { @@ -29,8 +33,41 @@ namespace fs = std::filesystem; class AssetsManager : public IdProvider { public: struct ResourceUpdates { + struct ModelUpdate { + ResourceId Id = 0; + HeadlessModel Model; + HeadlessModel::Header Header; + }; + + struct NodestateUpdate { + ResourceId Id = 0; + HeadlessNodeState Nodestate; + HeadlessNodeState::Header Header; + }; + + struct TextureUpdate { + ResourceId Id = 0; + uint16_t Width = 0; + uint16_t Height = 0; + std::vector Pixels; + std::string Domain; + std::string Key; + ResourceHeader Header; + }; + + struct BinaryUpdate { + ResourceId Id = 0; + std::u8string Data; + }; + + std::vector Models; + std::vector Nodestates; /// TODO: Добавить анимацию из меты - std::vector>> Textures; + std::vector Textures; + std::vector Particles; + std::vector Animations; + std::vector Sounds; + std::vector Fonts; }; public: @@ -148,6 +185,7 @@ public: static_cast(EnumAssets::MAX_ENUM) >& keys ) { + LOG.debug() << "BindDK domains=" << domains.size(); for(size_t type = 0; type < static_cast(EnumAssets::MAX_ENUM); ++type) { for(size_t forDomainIter = 0; forDomainIter < keys[type].size(); ++forDomainIter) { for(const std::string& key : keys[type][forDomainIter]) { @@ -164,21 +202,18 @@ public: static_cast(EnumAssets::MAX_ENUM) >&& hash_and_headers ) { - std::array< - std::vector>, - static_cast(EnumAssets::MAX_ENUM) - > hah = std::move(hash_and_headers); - std::unordered_set needHashes; + size_t totalBinds = 0; for(size_t type = 0; type < static_cast(EnumAssets::MAX_ENUM); ++type) { size_t maxSize = 0; for(auto& [id, hash, header] : hash_and_headers[type]) { + totalBinds++; assert(id < ServerToClientMap[type].size()); id = ServerToClientMap[type][id]; - if(id > maxSize) + if(id >= maxSize) maxSize = id+1; // Добавляем идентификатор в таблицу ожидающих обновлений. @@ -208,6 +243,9 @@ public: } } + if(totalBinds) + LOG.debug() << "BindHH total=" << totalBinds << " wait=" << WaitingHashes.size(); + // Нужно убрать хеши, которые уже запрошены // needHashes ^ WaitingHashes. @@ -251,8 +289,8 @@ public: vec.reserve(resources.size()); for(auto& [hash, res] : resources) { - vec.emplace_back(std::move(res)); - files.emplace(hash, res); + vec.emplace_back(res); + files.emplace(hash, std::move(res)); } _onHashLoad(files); @@ -260,7 +298,7 @@ public: } // Для запроса отсутствующих ресурсов с сервера на клиент. - std::vector pollNeededResources() { + std::vector pullNeededResources() { return std::move(NeedToRequestFromServer); } @@ -283,13 +321,35 @@ public: needToProceed.emplace(hash, std::u8string{(const char8_t*) res->data(), res->size()}); } + if(!NeedToRequestFromServer.empty()) + LOG.debug() << "CacheMiss count=" << NeedToRequestFromServer.size(); + if(!needToProceed.empty()) _onHashLoad(needToProceed); } - // Почитаем с диска + /// Читаем с диска TODO: получилась хрень с определением типа, чтобы получать headless ресурс if(!NeedToReadFromDisk.empty()) { std::unordered_map files; + files.reserve(NeedToReadFromDisk.size()); + + auto detectTypeDomainKey = [&](const fs::path& path, EnumAssets& typeOut, std::string& domainOut, std::string& keyOut) -> bool { + fs::path cur = path.parent_path(); + for(; !cur.empty(); cur = cur.parent_path()) { + std::string name = cur.filename().string(); + for(size_t typeIndex = 0; typeIndex < static_cast(EnumAssets::MAX_ENUM); ++typeIndex) { + EnumAssets type = static_cast(typeIndex); + if(name == ::EnumAssetsToDirectory(type)) { + typeOut = type; + domainOut = cur.parent_path().filename().string(); + keyOut = fs::relative(path, cur).generic_string(); + return true; + } + } + } + return false; + }; + for(const auto& [hash, path] : NeedToReadFromDisk) { std::u8string data; std::ifstream file(path, std::ios::binary); @@ -305,7 +365,50 @@ public: if(!file) data.clear(); } + } else { + LOG.warn() << "DiskReadFail " << path.string(); } + + if(!data.empty()) { + EnumAssets type{}; + std::string domain; + std::string key; + if(detectTypeDomainKey(path, type, domain, key)) { + if(type == EnumAssets::Nodestate) { + std::string_view view(reinterpret_cast(data.data()), data.size()); + js::object obj = js::parse(view).as_object(); + HeadlessNodeState hns; + auto modelResolver = [&](const std::string_view model) -> AssetsModel { + auto [mDomain, mKey] = parseDomainKey(model, domain); + return getId(EnumAssets::Model, mDomain, mKey); + }; + hns.parse(obj, modelResolver); + data = hns.dump(); + } else if(type == EnumAssets::Model) { + std::string_view view(reinterpret_cast(data.data()), data.size()); + js::object obj = js::parse(view).as_object(); + HeadlessModel hm; + auto modelResolver = [&](const std::string_view model) -> AssetsModel { + auto [mDomain, mKey] = parseDomainKey(model, domain); + return getId(EnumAssets::Model, mDomain, mKey); + }; + auto textureIdResolver = [&](const std::string_view texture) -> std::optional { + auto [tDomain, tKey] = parseDomainKey(texture, domain); + return getId(EnumAssets::Texture, tDomain, tKey); + }; + auto textureResolver = [&](const std::string_view texturePipelineSrc) -> std::vector { + TexturePipelineProgram tpp; + if(!tpp.compile(texturePipelineSrc)) + return {}; + tpp.link(textureIdResolver); + return tpp.toBytes(); + }; + hm.parse(obj, modelResolver, textureResolver); + data = hm.dump(); + } + } + } + files.emplace(hash, std::move(data)); } @@ -315,6 +418,8 @@ public: } private: + Logger LOG = "Client>AssetsManager"; + // Менеджеры учёта дисковых ресурсов AssetsPreloader // В приоритете ищутся ресурсы из ресурспаков по Domain+Key. @@ -364,7 +469,157 @@ private: // Когда данные были получены с диска, кеша или сервера void _onHashLoad(const std::unordered_map& files) { - /// TODO: скомпилировать ресурсы + const auto& rpLinks = ResourcePacks.getResourceLinks(); + const auto& esLinks = ExtraSource.getResourceLinks(); + + auto mapModelId = [&](ResourceId id) -> ResourceId { + const auto& map = ServerToClientMap[static_cast(EnumAssets::Model)]; + if(id >= map.size()) + return 0; + return map[id]; + }; + auto mapTextureId = [&](ResourceId id) -> ResourceId { + const auto& map = ServerToClientMap[static_cast(EnumAssets::Texture)]; + if(id >= map.size()) + return 0; + return map[id]; + }; + auto rebindHeader = [&](EnumAssets type, const ResourceHeader& header) -> ResourceHeader { + if(header.empty()) + return {}; + std::vector bytes; + bytes.resize(header.size()); + std::memcpy(bytes.data(), header.data(), header.size()); + std::vector rebound = AssetsHeaderCodec::rebindHeader( + type, + bytes, + mapModelId, + mapTextureId, + [](const std::string&) {} + ); + return ResourceHeader(reinterpret_cast(rebound.data()), rebound.size()); + }; + + for(size_t typeIndex = 0; typeIndex < static_cast(EnumAssets::MAX_ENUM); ++typeIndex) { + auto& pending = PendingUpdateFromAsync[typeIndex]; + if(pending.empty()) + continue; + + std::vector stillPending; + stillPending.reserve(pending.size()); + size_t updated = 0; + size_t missingSource = 0; + size_t missingData = 0; + + for(ResourceId id : pending) { + ResourceFile::Hash_t hash{}; + ResourceHeader header; + bool hasSource = false; + bool localHeader = false; + + if(id < rpLinks[typeIndex].size() && rpLinks[typeIndex][id].IsExist) { + hash = rpLinks[typeIndex][id].Hash; + header = rpLinks[typeIndex][id].Header; + hasSource = true; + localHeader = true; + } else if(id < ServerIdToHH[typeIndex].size()) { + std::tie(hash, header) = ServerIdToHH[typeIndex][id]; + hasSource = true; + } + + if(!hasSource) { + missingSource++; + stillPending.push_back(id); + continue; + } + + auto dataIter = files.find(hash); + if(dataIter == files.end()) { + missingData++; + stillPending.push_back(id); + continue; + } + + const auto& dkTable = IdToDK[typeIndex]; + std::string domain = "core"; + std::string key; + if(id < dkTable.size()) { + domain = dkTable[id].Domain; + key = dkTable[id].Key; + } + + std::u8string data = dataIter->second; + EnumAssets type = static_cast(typeIndex); + ResourceHeader finalHeader = localHeader ? header : rebindHeader(type, header); + + if(id == 0) + continue; + + if(type == EnumAssets::Nodestate) { + HeadlessNodeState ns; + ns.load(data); + HeadlessNodeState::Header headerParsed; + headerParsed.load(finalHeader); + RU.Nodestates.push_back({id, std::move(ns), std::move(headerParsed)}); + updated++; + } else if(type == EnumAssets::Model) { + HeadlessModel hm; + hm.load(data); + HeadlessModel::Header headerParsed; + headerParsed.load(finalHeader); + RU.Models.push_back({id, std::move(hm), std::move(headerParsed)}); + updated++; + } else if(type == EnumAssets::Texture) { + ResourceUpdates::TextureUpdate update; + update.Id = id; + update.Domain = std::move(domain); + update.Key = std::move(key); + update.Header = std::move(finalHeader); + if(!data.empty()) { + iResource sres(reinterpret_cast(data.data()), data.size()); + iBinaryStream stream = sres.makeStream(); + png::image img(stream.Stream); + update.Width = static_cast(img.get_width()); + update.Height = static_cast(img.get_height()); + update.Pixels.resize(static_cast(update.Width) * update.Height); + for(uint32_t y = 0; y < update.Height; ++y) { + const auto& row = img.get_pixbuf().operator[](y); + for(uint32_t x = 0; x < update.Width; ++x) { + const auto& px = row[x]; + uint32_t rgba = (uint32_t(px.alpha) << 24) + | (uint32_t(px.red) << 16) + | (uint32_t(px.green) << 8) + | uint32_t(px.blue); + update.Pixels[x + y * update.Width] = rgba; + } + } + } + RU.Textures.push_back(std::move(update)); + updated++; + } else if(type == EnumAssets::Particle) { + RU.Particles.push_back({id, std::move(data)}); + updated++; + } else if(type == EnumAssets::Animation) { + RU.Animations.push_back({id, std::move(data)}); + updated++; + } else if(type == EnumAssets::Sound) { + RU.Sounds.push_back({id, std::move(data)}); + updated++; + } else if(type == EnumAssets::Font) { + RU.Fonts.push_back({id, std::move(data)}); + updated++; + } + } + + if(updated || missingSource || missingData) { + LOG.debug() << "HashLoad type=" << int(typeIndex) + << " updated=" << updated + << " missingSource=" << missingSource + << " missingData=" << missingData; + } + + pending = std::move(stillPending); + } for(const auto& [hash, res] : files) WaitingHashes.erase(hash); diff --git a/Src/Client/ServerSession.cpp b/Src/Client/ServerSession.cpp index 030e149..4c48772 100644 --- a/Src/Client/ServerSession.cpp +++ b/Src/Client/ServerSession.cpp @@ -310,25 +310,27 @@ void ServerSession::update(GlobalTime gTime, float dTime) { // Если AssetsManager запрашивает ресурсы с сервера { std::vector needRequest = AM.pullNeededResources(); - Net::Packet pack; - std::vector packets; + if(!needRequest.empty()) { + Net::Packet pack; + std::vector packets; - auto check = [&]() { - if(pack.size() > 64000) + auto check = [&]() { + if(pack.size() > 64000) + packets.emplace_back(std::move(pack)); + }; + + pack << (uint8_t) ToServer::L1::System << (uint8_t) ToServer::L2System::ResourceRequest; + pack << (uint16_t) needRequest.size(); + for(const Hash_t& hash : needRequest) { + pack.write((const std::byte*) hash.data(), 32); + check(); + } + + if(pack.size()) packets.emplace_back(std::move(pack)); - }; - pack << (uint8_t) ToServer::L1::System << (uint8_t) ToServer::L2System::ResourceRequest; - pack << (uint16_t) needRequest.size(); - for(const Hash_t& hash : needRequest) { - pack.write((const std::byte*) hash.data(), 32); - check(); + Socket->pushPackets(&packets); } - - if(pack.size()) - packets.emplace_back(std::move(pack)); - - Socket->pushPackets(&packets); } if(!AsyncContext.TickSequence.get_read().empty()) { @@ -489,10 +491,10 @@ void ServerSession::update(GlobalTime gTime, float dTime) { } { - for(auto& [id, info] : data.Entity_AddOrChange) { - auto iter = std::lower_bound(entity_Lost.begin(), entity_Lost.end(), id); - if(iter != entity_Lost.end() && *iter == id) - entity_Lost.erase(iter); + for(auto& [id, info] : data.Entity_AddOrChange) { + auto iter = std::lower_bound(entity_Lost.begin(), entity_Lost.end(), id); + if(iter != entity_Lost.end() && *iter == id) + entity_Lost.erase(iter); entity_AddOrChange[id] = info; } @@ -504,13 +506,101 @@ void ServerSession::update(GlobalTime gTime, float dTime) { entity_Lost.insert(entity_Lost.end(), data.Entity_Lost.begin(), data.Entity_Lost.end()); std::sort(entity_Lost.begin(), entity_Lost.end()); auto eraseIter = std::unique(entity_Lost.begin(), entity_Lost.end()); - entity_Lost.erase(eraseIter, entity_Lost.end()); + entity_Lost.erase(eraseIter, entity_Lost.end()); + } + } + + { + AssetsManager::ResourceUpdates updates = AM.pullResourceUpdates(); + + if(!updates.Models.empty()) { + auto& map = Assets[EnumAssets::Model]; + for(auto& update : updates.Models) { + AssetEntry entry; + entry.Id = update.Id; + entry.Model = std::move(update.Model); + entry.ModelHeader = std::move(update.Header); + map[entry.Id] = std::move(entry); + result.Assets_ChangeOrAdd[EnumAssets::Model].push_back(update.Id); } } - for(auto& [id, _] : profile_Voxel_AddOrChange) - result.Profiles_ChangeOrAdd[EnumDefContent::Voxel].push_back(id); - result.Profiles_Lost[EnumDefContent::Voxel] = profile_Voxel_Lost; + if(!updates.Nodestates.empty()) { + auto& map = Assets[EnumAssets::Nodestate]; + for(auto& update : updates.Nodestates) { + AssetEntry entry; + entry.Id = update.Id; + entry.Nodestate = std::move(update.Nodestate); + entry.NodestateHeader = std::move(update.Header); + map[entry.Id] = std::move(entry); + result.Assets_ChangeOrAdd[EnumAssets::Nodestate].push_back(update.Id); + } + } + + if(!updates.Textures.empty()) { + auto& map = Assets[EnumAssets::Texture]; + for(auto& update : updates.Textures) { + AssetEntry entry; + entry.Id = update.Id; + entry.Domain = std::move(update.Domain); + entry.Key = std::move(update.Key); + entry.Width = update.Width; + entry.Height = update.Height; + entry.Pixels = std::move(update.Pixels); + entry.Header = std::move(update.Header); + map[entry.Id] = std::move(entry); + result.Assets_ChangeOrAdd[EnumAssets::Texture].push_back(update.Id); + } + } + + if(!updates.Particles.empty()) { + auto& map = Assets[EnumAssets::Particle]; + for(auto& update : updates.Particles) { + AssetEntry entry; + entry.Id = update.Id; + entry.Data = std::move(update.Data); + map[entry.Id] = std::move(entry); + result.Assets_ChangeOrAdd[EnumAssets::Particle].push_back(update.Id); + } + } + + if(!updates.Animations.empty()) { + auto& map = Assets[EnumAssets::Animation]; + for(auto& update : updates.Animations) { + AssetEntry entry; + entry.Id = update.Id; + entry.Data = std::move(update.Data); + map[entry.Id] = std::move(entry); + result.Assets_ChangeOrAdd[EnumAssets::Animation].push_back(update.Id); + } + } + + if(!updates.Sounds.empty()) { + auto& map = Assets[EnumAssets::Sound]; + for(auto& update : updates.Sounds) { + AssetEntry entry; + entry.Id = update.Id; + entry.Data = std::move(update.Data); + map[entry.Id] = std::move(entry); + result.Assets_ChangeOrAdd[EnumAssets::Sound].push_back(update.Id); + } + } + + if(!updates.Fonts.empty()) { + auto& map = Assets[EnumAssets::Font]; + for(auto& update : updates.Fonts) { + AssetEntry entry; + entry.Id = update.Id; + entry.Data = std::move(update.Data); + map[entry.Id] = std::move(entry); + result.Assets_ChangeOrAdd[EnumAssets::Font].push_back(update.Id); + } + } + } + + for(auto& [id, _] : profile_Voxel_AddOrChange) + result.Profiles_ChangeOrAdd[EnumDefContent::Voxel].push_back(id); + result.Profiles_Lost[EnumDefContent::Voxel] = profile_Voxel_Lost; for(auto& [id, _] : profile_Node_AddOrChange) result.Profiles_ChangeOrAdd[EnumDefContent::Node].push_back(id); @@ -759,6 +849,8 @@ void ServerSession::update(GlobalTime gTime, float dTime) { RS->tickSync(result); } + AM.tick(); + // Здесь нужно обработать управляющие пакеты diff --git a/Src/Client/Vulkan/VulkanRenderSession.cpp b/Src/Client/Vulkan/VulkanRenderSession.cpp index 9f48d8c..dfbfe7a 100644 --- a/Src/Client/Vulkan/VulkanRenderSession.cpp +++ b/Src/Client/Vulkan/VulkanRenderSession.cpp @@ -1680,7 +1680,7 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) { if(auto iter = data.Profiles_Lost.find(EnumDefContent::Voxel); iter != data.Profiles_Lost.end()) mcpData.ChangedVoxels.insert(mcpData.ChangedVoxels.end(), iter->second.begin(), iter->second.end()); - std::vector*>> modelResources; + std::vector modelResources; std::vector modelLost; if(auto iter = data.Assets_ChangeOrAdd.find(EnumAssets::Model); iter != data.Assets_ChangeOrAdd.end()) { const auto& list = ServerSession->Assets[EnumAssets::Model]; @@ -1689,7 +1689,7 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) { if(entryIter == list.end()) continue; - modelResources.emplace_back(id, entryIter->second.Res, &entryIter->second.Dependencies); + modelResources.push_back(&entryIter->second); } } if(auto iter = data.Assets_Lost.find(EnumAssets::Model); iter != data.Assets_Lost.end()) @@ -1714,7 +1714,9 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) { textureResources.push_back({ .Id = id, - .Res = entryIter->second.Res, + .Width = entryIter->second.Width, + .Height = entryIter->second.Height, + .Pixels = entryIter->second.Pixels, .Domain = entryIter->second.Domain, .Key = entryIter->second.Key }); @@ -1730,7 +1732,7 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) { std::vector changedNodestates; if(NSP) { - std::vector*>> nodestateResources; + std::vector nodestateResources; std::vector nodestateLost; if(auto iter = data.Assets_ChangeOrAdd.find(EnumAssets::Nodestate); iter != data.Assets_ChangeOrAdd.end()) { @@ -1740,7 +1742,7 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) { if(entryIter == list.end()) continue; - nodestateResources.emplace_back(id, entryIter->second.Res, &entryIter->second.Dependencies); + nodestateResources.push_back(&entryIter->second); } } diff --git a/Src/Client/Vulkan/VulkanRenderSession.hpp b/Src/Client/Vulkan/VulkanRenderSession.hpp index 2fe73d1..b65a868 100644 --- a/Src/Client/Vulkan/VulkanRenderSession.hpp +++ b/Src/Client/Vulkan/VulkanRenderSession.hpp @@ -89,10 +89,11 @@ public: } // Применяет изменения, возвращая все затронутые модели - std::vector onModelChanges(std::vector*>> newOrChanged, + std::vector onModelChanges(std::vector newOrChanged, std::vector lost, const std::unordered_map* modelAssets) { std::vector result; + (void)modelAssets; std::move_only_function makeUnready; makeUnready = [&](ResourceId id) { @@ -136,87 +137,52 @@ public: Models.erase(iterModel); } - for(const auto& [key, resource, deps] : newOrChanged) { + for(const AssetEntry* entry : newOrChanged) { + if(!entry) + continue; + const AssetsModel key = entry->Id; result.push_back(key); makeUnready(key); ModelObject model; - std::string type = "unknown"; - std::optional header; - if(deps && !deps->empty()) - header = AssetsManager::parseHeader(EnumAssets::Model, *deps); - const std::vector* textureDeps = header ? &header->TextureDeps : nullptr; - auto remapTextureId = [&](uint32_t placeholder) -> uint32_t { - if(!textureDeps || placeholder >= textureDeps->size()) - return 0; - return (*textureDeps)[placeholder]; - }; - auto remapPipeline = [&](TexturePipeline pipe) { - if(textureDeps) { - for(auto& texId : pipe.BinTextures) - texId = remapTextureId(texId); - if(!pipe.Pipeline.empty()) { - std::vector code; - code.resize(pipe.Pipeline.size()); - std::memcpy(code.data(), pipe.Pipeline.data(), code.size()); - TexturePipelineProgram::remapTexIds(code, *textureDeps, nullptr); - pipe.Pipeline.resize(code.size()); - std::memcpy(pipe.Pipeline.data(), code.data(), code.size()); - } - } - return pipe; - }; + const HeadlessModel& hm = entry->Model; + const HeadlessModel::Header& header = entry->ModelHeader; - size_t dataSize = 0; - std::array prefix = {}; try { - std::u8string_view data((const char8_t*) resource.data(), resource.size()); - dataSize = data.size(); - if(!data.empty()) { - const size_t prefixLen = std::min(prefix.size(), data.size()); - for(size_t i = 0; i < prefixLen; ++i) - prefix[i] = static_cast(data[i]); - } - if(data.starts_with((const char8_t*) "bm")) { - type = "InternalBinary"; - // Компилированная модель внутреннего формата - HeadlessModel hm; - hm.load(data); - model.TextureMap.clear(); - model.TextureMap.reserve(hm.Textures.size()); - for(const auto& [tkey, id] : hm.Textures) { - TexturePipeline pipe; - if(header && id < header->TexturePipelines.size()) { - pipe.Pipeline = header->TexturePipelines[id]; - } else { - LOG.warn() << "Model texture pipeline id out of range: model=" << key - << " local=" << id - << " pipelines=" << (header ? header->TexturePipelines.size() : 0); - pipe.BinTextures.push_back(id); - pipe = remapPipeline(std::move(pipe)); - } - model.TextureMap.emplace(tkey, std::move(pipe)); + model.TextureMap.clear(); + model.TextureMap.reserve(hm.Textures.size()); + for(const auto& [tkey, id] : hm.Textures) { + TexturePipeline pipe; + if(id < header.TexturePipelines.size()) { + pipe.Pipeline = header.TexturePipelines[id]; + } else { + LOG.warn() << "Model texture pipeline id out of range: model=" << key + << " local=" << id + << " pipelines=" << header.TexturePipelines.size(); + pipe.BinTextures.push_back(id); } - model.TextureKeys = {}; + model.TextureMap.emplace(tkey, std::move(pipe)); + } + model.TextureKeys = {}; - for(const HeadlessModel::Cuboid& cb : hm.Cuboids) { - glm::vec3 min = glm::min(cb.From, cb.To), max = glm::max(cb.From, cb.To); + for(const HeadlessModel::Cuboid& cb : hm.Cuboids) { + glm::vec3 min = glm::min(cb.From, cb.To), max = glm::max(cb.From, cb.To); - for(const auto& [face, params] : cb.Faces) { - glm::vec2 from_uv = {params.UV[0], params.UV[1]}, to_uv = {params.UV[2], params.UV[3]}; + for(const auto& [face, params] : cb.Faces) { + glm::vec2 from_uv = {params.UV[0], params.UV[1]}, to_uv = {params.UV[2], params.UV[3]}; - uint32_t texId; - { - auto iter = std::find(model.TextureKeys.begin(), model.TextureKeys.end(), params.Texture); - if(iter == model.TextureKeys.end()) { - texId = model.TextureKeys.size(); - model.TextureKeys.push_back(params.Texture); - } else { - texId = iter-model.TextureKeys.begin(); - } + uint32_t texId; + { + auto iter = std::find(model.TextureKeys.begin(), model.TextureKeys.end(), params.Texture); + if(iter == model.TextureKeys.end()) { + texId = model.TextureKeys.size(); + model.TextureKeys.push_back(params.Texture); + } else { + texId = iter-model.TextureKeys.begin(); } + } - std::vector v; + std::vector v; auto addQuad = [&](const glm::vec3& p0, const glm::vec3& p1, @@ -277,20 +243,20 @@ public: cb.Trs.apply(v); model.Vertecies[params.Cullface].append_range(v); } - } + } - if(!hm.SubModels.empty()) { - model.Depends.reserve(hm.SubModels.size()); - for(const auto& sub : hm.SubModels) { - if(!header || sub.Id >= header->ModelDeps.size()) { - LOG.warn() << "Model sub-model id out of range: model=" << key - << " local=" << sub.Id - << " deps=" << (header ? header->ModelDeps.size() : 0); - continue; - } - model.Depends.emplace_back(header->ModelDeps[sub.Id], Transformations{}); + if(!hm.SubModels.empty()) { + model.Depends.reserve(hm.SubModels.size()); + for(const auto& sub : hm.SubModels) { + if(sub.Id >= header.Models.size()) { + LOG.warn() << "Model sub-model id out of range: model=" << key + << " local=" << sub.Id + << " deps=" << header.Models.size(); + continue; } + model.Depends.emplace_back(header.Models[sub.Id], Transformations{}); } + } // struct Face { // int TintIndex = -1; @@ -299,18 +265,8 @@ public: // std::vector Transformations; - } else if(data.starts_with((const char8_t*) "glTF")) { - type = "glb"; - - } else if(data.starts_with((const char8_t*) "bgl")) { - type = "InternalGLTF"; - - } else if(data.starts_with((const char8_t*) "{")) { - type = "InternalJson или glTF"; - // Модель внутреннего формата или glTF - } } catch(const std::exception& exc) { - LOG.warn() << "Не удалось распарсить модель " << type << ":\n\t" << exc.what(); + LOG.warn() << "Не удалось собрать модель:\n\t" << exc.what(); continue; } @@ -321,11 +277,10 @@ public: size_t vertexCount = 0; for(const auto& [_, verts] : model.Vertecies) vertexCount += verts.size(); - size_t texDepsCount = textureDeps ? textureDeps->size() : 0; LOG.debug() << "Model loaded id=" << key << " verts=" << vertexCount << " texKeys=" << model.TextureKeys.size() - << " texDeps=" << texDepsCount; + << " pipelines=" << header.TexturePipelines.size(); } } @@ -334,12 +289,7 @@ public: uint32_t idx = debugEmptyModelLogCount.fetch_add(1); if(idx < 128) { LOG.warn() << "Model has empty geometry id=" << key - << " type=" << type - << " size=" << dataSize - << " prefix=" << int(prefix[0]) << '.' - << int(prefix[1]) << '.' - << int(prefix[2]) << '.' - << int(prefix[3]); + << " pipelines=" << header.TexturePipelines.size(); } } @@ -495,7 +445,9 @@ class TextureProvider { public: struct TextureUpdate { AssetsTexture Id = 0; - Resource Res; + uint16_t Width = 0; + uint16_t Height = 0; + std::vector Pixels; std::string Domain; std::string Key; }; @@ -639,40 +591,21 @@ public: std::lock_guard lock(Mutex); std::vector result; - for(const auto& update : newOrChanged) { + for(auto& update : newOrChanged) { const AssetsTexture key = update.Id; - const Resource& res = update.Res; result.push_back(key); - iResource sres((const uint8_t*) res.data(), res.size()); - iBinaryStream stream = sres.makeStream(); - png::image img(stream.Stream); - uint32_t width = img.get_width(); - uint32_t height = img.get_height(); - - std::vector pixels; - pixels.resize(width*height); - - for(uint32_t y = 0; y < height; y++) { - const auto& row = img.get_pixbuf().operator [](y); - for(uint32_t x = 0; x < width; x++) { - const auto& px = row[x]; - uint32_t rgba = (uint32_t(px.alpha) << 24) - | (uint32_t(px.red) << 16) - | (uint32_t(px.green) << 8) - | uint32_t(px.blue); - pixels[x + y * width] = rgba; - } - } + if(update.Width == 0 || update.Height == 0 || update.Pixels.empty()) + continue; Atlas->updateTexture(key, StoredTexture( - static_cast(width), - static_cast(height), - std::move(pixels) + update.Width, + update.Height, + std::move(update.Pixels) )); bool animated = false; - if(auto anim = getDefaultAnimation(update.Key, width, height)) { + if(auto anim = getDefaultAnimation(update.Key, update.Width, update.Height)) { AnimatedSources[key] = *anim; animated = true; } else { @@ -686,7 +619,7 @@ public: if(idx < 128) { LOG.debug() << "Texture loaded id=" << key << " key=" << update.Domain << ':' << update.Key - << " size=" << width << 'x' << height + << " size=" << update.Width << 'x' << update.Height << " animated=" << (animated ? 1 : 0); } } @@ -911,7 +844,7 @@ public: {} // Применяет изменения, возвращает изменённые описания состояний - std::vector onNodestateChanges(std::vector*>> newOrChanged, std::vector lost, std::vector changedModels) { + std::vector onNodestateChanges(std::vector newOrChanged, std::vector lost, std::vector changedModels) { std::vector result; for(ResourceId lostId : lost) { @@ -923,42 +856,15 @@ public: Nodestates.erase(iterNodestate); } - for(const auto& [key, resource, deps] : newOrChanged) { + for(const AssetEntry* entry : newOrChanged) { + if(!entry) + continue; + const AssetsNodestate key = entry->Id; result.push_back(key); PreparedNodeState nodestate; - std::string type = "unknown"; - - try { - std::u8string_view data((const char8_t*) resource.data(), resource.size()); - if(data.starts_with((const char8_t*) "bn")) { - type = "InternalBinary"; - // Компилированный nodestate внутреннего формата - nodestate = PreparedNodeState(data); - } else if(data.starts_with((const char8_t*) "{")) { - type = "InternalJson"; - // nodestate в json формате - } else { - type = "InternalBinaryLegacy"; - // Старый двоичный формат без заголовка "bn" - std::u8string patched; - patched.reserve(data.size() + 2); - patched.push_back(u8'b'); - patched.push_back(u8'n'); - patched.append(data); - nodestate = PreparedNodeState(patched); - } - } catch(const std::exception& exc) { - LOG.warn() << "Не удалось распарсить nodestate " << type << ":\n\t" << exc.what(); - continue; - } - - if(deps && !deps->empty()) { - auto header = AssetsManager::parseHeader(EnumAssets::Nodestate, *deps); - if(header && header->Type == EnumAssets::Nodestate) { - nodestate.LocalToModel.assign(header->ModelDeps.begin(), header->ModelDeps.end()); - } - } + static_cast(nodestate) = entry->Nodestate; + nodestate.LocalToModel.assign(entry->NodestateHeader.Models.begin(), entry->NodestateHeader.Models.end()); Nodestates.insert_or_assign(key, std::move(nodestate)); if(key < 64) { diff --git a/Src/Common/Abstract.cpp b/Src/Common/Abstract.cpp index 0990344..37a9454 100644 --- a/Src/Common/Abstract.cpp +++ b/Src/Common/Abstract.cpp @@ -828,19 +828,117 @@ Hash_t ResourceFile::calcHash(const char8_t* data, size_t size) { return sha2::sha256((const uint8_t*) data, size); } +uint16_t HeadlessNodeState::Header::addModel(AssetsModel id) { + auto iter = std::find(Models.begin(), Models.end(), id); + if(iter == Models.end()) { + Models.push_back(id); + return Models.size() - 1; + } + + return iter - Models.begin(); +} + +void HeadlessNodeState::Header::load(std::u8string_view data) { + Models.clear(); + if(data.empty()) + return; + if(data.size() % sizeof(AssetsModel) != 0) + MAKE_ERROR("Invalid nodestate header size"); + + const size_t count = data.size() / sizeof(AssetsModel); + Models.resize(count); + std::memcpy(Models.data(), data.data(), data.size()); +} + +ResourceHeader HeadlessNodeState::Header::dump() const { + ResourceHeader rh; + rh.reserve(Models.size() * sizeof(AssetsModel)); + + for(AssetsModel id : Models) { + rh += std::u8string_view((const char8_t*) &id, sizeof(AssetsModel)); + } + + return rh; +} + +uint16_t HeadlessModel::Header::addModel(AssetsModel id) { + auto iter = std::find(Models.begin(), Models.end(), id); + if(iter == Models.end()) { + Models.push_back(id); + return Models.size() - 1; + } + + return iter - Models.begin(); +} + +uint16_t HeadlessModel::Header::addTexturePipeline(std::vector pipeline) { + TexturePipelines.push_back(std::move(pipeline)); + return TexturePipelines.size() - 1; +} + +void HeadlessModel::Header::load(std::u8string_view data) { + Models.clear(); + TexturePipelines.clear(); + if(data.empty()) + return; + + try { + TOS::ByteBuffer buffer(data.size(), reinterpret_cast(data.data())); + auto reader = buffer.reader(); + + uint16_t modelCount = reader.readUInt16(); + Models.reserve(modelCount); + for(uint16_t i = 0; i < modelCount; ++i) + Models.push_back(reader.readUInt32()); + + uint16_t texCount = reader.readUInt16(); + TexturePipelines.reserve(texCount); + for(uint16_t i = 0; i < texCount; ++i) { + uint32_t size32 = reader.readUInt32(); + TOS::ByteBuffer pipe; + reader.readBuffer(pipe); + if(pipe.size() != size32) + MAKE_ERROR("Invalid model header size"); + TexturePipelines.emplace_back(pipe.begin(), pipe.end()); + } + } catch(const std::exception&) { + MAKE_ERROR("Invalid model header"); + } +} + +ResourceHeader HeadlessModel::Header::dump() const { + TOS::ByteBuffer rh; + + { + uint32_t fullSize = 0; + for(const auto& vector : TexturePipelines) + fullSize += vector.size(); + rh.reserve(2 + Models.size() * sizeof(AssetsModel) + 2 + 4 * TexturePipelines.size() + fullSize); + } + + TOS::ByteBuffer::Writer wr; + wr << uint16_t(Models.size()); + for(AssetsModel id : Models) + wr << id; + + wr << uint16_t(TexturePipelines.size()); + for(const auto& pipe : TexturePipelines) { + wr << uint32_t(pipe.size()); + wr << pipe; + } + + TOS::ByteBuffer buff = wr.complite(); + + return std::u8string((const char8_t*) buff.data(), buff.size()); +} + ResourceHeader HeadlessNodeState::parse(const js::object& profile, const std::function& modelResolver) { - std::vector headerIds; + Header header; std::function headerResolver = [&](const std::string_view model) -> uint16_t { AssetsModel id = modelResolver(model); - auto iter = std::find(headerIds.begin(), headerIds.end(), id); - if(iter == headerIds.end()) { - headerIds.push_back(id); - return headerIds.size()-1; - } - - return iter-headerIds.begin(); + return header.addModel(id); }; for(auto& [condition, variability] : profile) { @@ -869,14 +967,7 @@ ResourceHeader HeadlessNodeState::parse(const js::object& profile, const std::fu Routes.emplace_back(node, std::move(models)); } - ResourceHeader rh; - rh.reserve(headerIds.size()*sizeof(AssetsModel)); - - for(AssetsModel id : headerIds) { - rh += std::u8string_view((const char8_t*) &id, sizeof(AssetsModel)); - } - - return rh; + return header.dump(); } ResourceHeader HeadlessNodeState::parse(const sol::table& profile, const std::function& modelResolver) { @@ -1494,21 +1585,14 @@ ResourceHeader HeadlessModel::parse( const std::function& modelResolver, const std::function(const std::string_view texturePipelineSrc)>& textureResolver ) { - std::vector headerIdsModels; + Header header; std::function headerResolverModel = [&](const std::string_view model) -> uint16_t { AssetsModel id = modelResolver(model); - auto iter = std::find(headerIdsModels.begin(), headerIdsModels.end(), id); - if(iter == headerIdsModels.end()) { - headerIdsModels.push_back(id); - return headerIdsModels.size()-1; - } - - return iter-headerIdsModels.begin(); + return header.addModel(id); }; - std::vector> headerIdsTextures; std::unordered_map textureToLocal; std::function headerResolverTexture = @@ -1518,9 +1602,8 @@ ResourceHeader HeadlessModel::parse( return iter->second; } - std::vector program = textureResolver(texturePipelineSrc); - headerIdsTextures.push_back(program); - uint16_t id = textureToLocal[(std::string) texturePipelineSrc] = headerIdsTextures.size()-1; + uint16_t id = header.addTexturePipeline(textureResolver(texturePipelineSrc)); + textureToLocal[(std::string) texturePipelineSrc] = id; return id; }; @@ -1748,27 +1831,7 @@ ResourceHeader HeadlessModel::parse( // Заголовок TOS::ByteBuffer rh; - { - uint32_t fullSize = 0; - for(const auto& vector : headerIdsTextures) - fullSize += vector.size(); - rh.reserve(2+headerIdsModels.size()*sizeof(AssetsModel)+2+(4)*headerIdsTextures.size()+fullSize); - } - - TOS::ByteBuffer::Writer wr; - wr << uint16_t(headerIdsModels.size()); - for(AssetsModel id : headerIdsModels) - wr << id; - - wr << uint16_t(headerIdsTextures.size()); - for(const auto& pipe : headerIdsTextures) { - wr << uint32_t(pipe.size()); - wr << pipe; - } - - TOS::ByteBuffer buff = wr.complite(); - - return std::u8string((const char8_t*) buff.data(), buff.size()); + return header.dump(); } ResourceHeader HeadlessModel::parse( diff --git a/Src/Common/Abstract.hpp b/Src/Common/Abstract.hpp index 8f17bea..7093b97 100644 --- a/Src/Common/Abstract.hpp +++ b/Src/Common/Abstract.hpp @@ -675,6 +675,14 @@ struct HeadlessNodeState { std::vector Transforms; }; + struct Header { + std::vector Models; + + uint16_t addModel(AssetsModel id); + void load(std::u8string_view data); + ResourceHeader dump() const; + }; + // Ноды выражений std::vector Nodes; // Условия -> вариации модели + веса @@ -911,6 +919,16 @@ struct HeadlessModel { std::optional GuiLight = EnumGuiLight::Default; std::optional AmbientOcclusion = false; + + struct Header { + std::vector Models; + std::vector> TexturePipelines; + + uint16_t addModel(AssetsModel id); + uint16_t addTexturePipeline(std::vector pipeline); + void load(std::u8string_view data); + ResourceHeader dump() const; + }; struct FullTransformation { glm::vec3