DrSocalkwe3n/LuaVox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

codex-5.2: тест новой версии менеджера ассетов

Browse Source
This commit is contained in:
DrSocalkwe3n
2026-01-07 04:03:17 +06:00
parent 523f9725c0
commit 5135aa30a7
7 changed files with 607 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
Src/Client/Abstract.hpp
View File

@@ -158,6 +158,25 @@ struct DefNode_t {
};
struct AssetEntry {
ResourceId Id = 0;
std::string Domain;
std::string Key;
HeadlessModel Model;
HeadlessModel::Header ModelHeader;
HeadlessNodeState Nodestate;
HeadlessNodeState::Header NodestateHeader;
uint16_t Width = 0;
uint16_t Height = 0;
std::vector<uint32_t> Pixels;
ResourceHeader Header;
std::u8string Data;
};
/*
Интерфейс обработчика сессии с сервером.
@@ -172,7 +191,7 @@ public:
bool DebugLogPackets = false;
// Используемые двоичные ресурсы
// std::unordered_map<EnumAssets, std::unordered_map<ResourceId, AssetEntry>> Assets;
std::unordered_map<EnumAssets, std::unordered_map<ResourceId, AssetEntry>> Assets;
// Используемые профили контента
struct {

279
Src/Client/AssetsManager.hpp
View File

@@ -13,13 +13,17 @@
#include <unordered_set>
#include <utility>
#include <vector>
#include <cstring>
#include "Client/AssetsCacheManager.hpp"
#include "Client/AssetsHeaderCodec.hpp"
#include "Common/Abstract.hpp"
#include "Common/IdProvider.hpp"
#include "Common/AssetsPreloader.hpp"
#include "Common/TexturePipelineProgram.hpp"
#include "TOSLib.hpp"
#include "assets.hpp"
#include "boost/asio/io_context.hpp"
#include "png++/image.hpp"
#include <fstream>
namespace LV::Client {
@@ -29,8 +33,41 @@ namespace fs = std::filesystem;
class AssetsManager : public IdProvider<EnumAssets> {
public:
struct ResourceUpdates {
struct ModelUpdate {
ResourceId Id = 0;
HeadlessModel Model;
HeadlessModel::Header Header;
};
struct NodestateUpdate {
ResourceId Id = 0;
HeadlessNodeState Nodestate;
HeadlessNodeState::Header Header;
};
struct TextureUpdate {
ResourceId Id = 0;
uint16_t Width = 0;
uint16_t Height = 0;
std::vector<uint32_t> Pixels;
std::string Domain;
std::string Key;
ResourceHeader Header;
};
struct BinaryUpdate {
ResourceId Id = 0;
std::u8string Data;
};
std::vector<ModelUpdate> Models;
std::vector<NodestateUpdate> Nodestates;
/// TODO: Добавить анимацию из меты
std::vector<std::tuple<ResourceId, uint16_t, uint16_t, std::vector<uint32_t>>> Textures;
std::vector<TextureUpdate> Textures;
std::vector<BinaryUpdate> Particles;
std::vector<BinaryUpdate> Animations;
std::vector<BinaryUpdate> Sounds;
std::vector<BinaryUpdate> Fonts;
};
public:
@@ -148,6 +185,7 @@ public:
static_cast<size_t>(EnumAssets::MAX_ENUM)
>& keys
) {
LOG.debug() << "BindDK domains=" << domains.size();
for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); ++type) {
for(size_t forDomainIter = 0; forDomainIter < keys[type].size(); ++forDomainIter) {
for(const std::string& key : keys[type][forDomainIter]) {
@@ -164,21 +202,18 @@ public:
static_cast<size_t>(EnumAssets::MAX_ENUM)
>&& hash_and_headers
) {
std::array<
std::vector<std::tuple<ResourceId, ResourceFile::Hash_t, ResourceHeader>>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
> hah = std::move(hash_and_headers);
std::unordered_set<ResourceFile::Hash_t> needHashes;
size_t totalBinds = 0;
for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); ++type) {
size_t maxSize = 0;
for(auto& [id, hash, header] : hash_and_headers[type]) {
totalBinds++;
assert(id < ServerToClientMap[type].size());
id = ServerToClientMap[type][id];
if(id > maxSize)
if(id >= maxSize)
maxSize = id+1;
// Добавляем идентификатор в таблицу ожидающих обновлений.
@@ -208,6 +243,9 @@ public:
}
}
if(totalBinds)
LOG.debug() << "BindHH total=" << totalBinds << " wait=" << WaitingHashes.size();
// Нужно убрать хеши, которые уже запрошены
// needHashes ^ WaitingHashes.
@@ -251,8 +289,8 @@ public:
vec.reserve(resources.size());
for(auto& [hash, res] : resources) {
vec.emplace_back(std::move(res));
files.emplace(hash, res);
vec.emplace_back(res);
files.emplace(hash, std::move(res));
}
_onHashLoad(files);
@@ -260,7 +298,7 @@ public:
}
// Для запроса отсутствующих ресурсов с сервера на клиент.
std::vector<ResourceFile::Hash_t> pollNeededResources() {
std::vector<ResourceFile::Hash_t> pullNeededResources() {
return std::move(NeedToRequestFromServer);
}
@@ -283,13 +321,35 @@ public:
needToProceed.emplace(hash, std::u8string{(const char8_t*) res->data(), res->size()});
}
if(!NeedToRequestFromServer.empty())
LOG.debug() << "CacheMiss count=" << NeedToRequestFromServer.size();
if(!needToProceed.empty())
_onHashLoad(needToProceed);
}
// Почитаем с диска
/// Читаем с диска TODO: получилась хрень с определением типа, чтобы получать headless ресурс
if(!NeedToReadFromDisk.empty()) {
std::unordered_map<ResourceFile::Hash_t, std::u8string> files;
files.reserve(NeedToReadFromDisk.size());
auto detectTypeDomainKey = [&](const fs::path& path, EnumAssets& typeOut, std::string& domainOut, std::string& keyOut) -> bool {
fs::path cur = path.parent_path();
for(; !cur.empty(); cur = cur.parent_path()) {
std::string name = cur.filename().string();
for(size_t typeIndex = 0; typeIndex < static_cast<size_t>(EnumAssets::MAX_ENUM); ++typeIndex) {
EnumAssets type = static_cast<EnumAssets>(typeIndex);
if(name == ::EnumAssetsToDirectory(type)) {
typeOut = type;
domainOut = cur.parent_path().filename().string();
keyOut = fs::relative(path, cur).generic_string();
return true;
}
}
}
return false;
};
for(const auto& [hash, path] : NeedToReadFromDisk) {
std::u8string data;
std::ifstream file(path, std::ios::binary);
@@ -305,7 +365,50 @@ public:
if(!file)
data.clear();
}
} else {
LOG.warn() << "DiskReadFail " << path.string();
}
if(!data.empty()) {
EnumAssets type{};
std::string domain;
std::string key;
if(detectTypeDomainKey(path, type, domain, key)) {
if(type == EnumAssets::Nodestate) {
std::string_view view(reinterpret_cast<const char*>(data.data()), data.size());
js::object obj = js::parse(view).as_object();
HeadlessNodeState hns;
auto modelResolver = [&](const std::string_view model) -> AssetsModel {
auto [mDomain, mKey] = parseDomainKey(model, domain);
return getId(EnumAssets::Model, mDomain, mKey);
};
hns.parse(obj, modelResolver);
data = hns.dump();
} else if(type == EnumAssets::Model) {
std::string_view view(reinterpret_cast<const char*>(data.data()), data.size());
js::object obj = js::parse(view).as_object();
HeadlessModel hm;
auto modelResolver = [&](const std::string_view model) -> AssetsModel {
auto [mDomain, mKey] = parseDomainKey(model, domain);
return getId(EnumAssets::Model, mDomain, mKey);
};
auto textureIdResolver = [&](const std::string_view texture) -> std::optional<uint32_t> {
auto [tDomain, tKey] = parseDomainKey(texture, domain);
return getId(EnumAssets::Texture, tDomain, tKey);
};
auto textureResolver = [&](const std::string_view texturePipelineSrc) -> std::vector<uint8_t> {
TexturePipelineProgram tpp;
if(!tpp.compile(texturePipelineSrc))
return {};
tpp.link(textureIdResolver);
return tpp.toBytes();
};
hm.parse(obj, modelResolver, textureResolver);
data = hm.dump();
}
}
}
files.emplace(hash, std::move(data));
}
@@ -315,6 +418,8 @@ public:
}
private:
Logger LOG = "Client>AssetsManager";
// Менеджеры учёта дисковых ресурсов
AssetsPreloader
// В приоритете ищутся ресурсы из ресурспаков по Domain+Key.
@@ -364,7 +469,157 @@ private:
// Когда данные были получены с диска, кеша или сервера
void _onHashLoad(const std::unordered_map<ResourceFile::Hash_t, std::u8string>& files) {
/// TODO: скомпилировать ресурсы
const auto& rpLinks = ResourcePacks.getResourceLinks();
const auto& esLinks = ExtraSource.getResourceLinks();
auto mapModelId = [&](ResourceId id) -> ResourceId {
const auto& map = ServerToClientMap[static_cast<size_t>(EnumAssets::Model)];
if(id >= map.size())
return 0;
return map[id];
};
auto mapTextureId = [&](ResourceId id) -> ResourceId {
const auto& map = ServerToClientMap[static_cast<size_t>(EnumAssets::Texture)];
if(id >= map.size())
return 0;
return map[id];
};
auto rebindHeader = [&](EnumAssets type, const ResourceHeader& header) -> ResourceHeader {
if(header.empty())
return {};
std::vector<uint8_t> bytes;
bytes.resize(header.size());
std::memcpy(bytes.data(), header.data(), header.size());
std::vector<uint8_t> rebound = AssetsHeaderCodec::rebindHeader(
type,
bytes,
mapModelId,
mapTextureId,
[](const std::string&) {}
);
return ResourceHeader(reinterpret_cast<const char8_t*>(rebound.data()), rebound.size());
};
for(size_t typeIndex = 0; typeIndex < static_cast<size_t>(EnumAssets::MAX_ENUM); ++typeIndex) {
auto& pending = PendingUpdateFromAsync[typeIndex];
if(pending.empty())
continue;
std::vector<ResourceId> stillPending;
stillPending.reserve(pending.size());
size_t updated = 0;
size_t missingSource = 0;
size_t missingData = 0;
for(ResourceId id : pending) {
ResourceFile::Hash_t hash{};
ResourceHeader header;
bool hasSource = false;
bool localHeader = false;
if(id < rpLinks[typeIndex].size() && rpLinks[typeIndex][id].IsExist) {
hash = rpLinks[typeIndex][id].Hash;
header = rpLinks[typeIndex][id].Header;
hasSource = true;
localHeader = true;
} else if(id < ServerIdToHH[typeIndex].size()) {
std::tie(hash, header) = ServerIdToHH[typeIndex][id];
hasSource = true;
}
if(!hasSource) {
missingSource++;
stillPending.push_back(id);
continue;
}
auto dataIter = files.find(hash);
if(dataIter == files.end()) {
missingData++;
stillPending.push_back(id);
continue;
}
const auto& dkTable = IdToDK[typeIndex];
std::string domain = "core";
std::string key;
if(id < dkTable.size()) {
domain = dkTable[id].Domain;
key = dkTable[id].Key;
}
std::u8string data = dataIter->second;
EnumAssets type = static_cast<EnumAssets>(typeIndex);
ResourceHeader finalHeader = localHeader ? header : rebindHeader(type, header);
if(id == 0)
continue;
if(type == EnumAssets::Nodestate) {
HeadlessNodeState ns;
ns.load(data);
HeadlessNodeState::Header headerParsed;
headerParsed.load(finalHeader);
RU.Nodestates.push_back({id, std::move(ns), std::move(headerParsed)});
updated++;
} else if(type == EnumAssets::Model) {
HeadlessModel hm;
hm.load(data);
HeadlessModel::Header headerParsed;
headerParsed.load(finalHeader);
RU.Models.push_back({id, std::move(hm), std::move(headerParsed)});
updated++;
} else if(type == EnumAssets::Texture) {
ResourceUpdates::TextureUpdate update;
update.Id = id;
update.Domain = std::move(domain);
update.Key = std::move(key);
update.Header = std::move(finalHeader);
if(!data.empty()) {
iResource sres(reinterpret_cast<const uint8_t*>(data.data()), data.size());
iBinaryStream stream = sres.makeStream();
png::image<png::rgba_pixel> img(stream.Stream);
update.Width = static_cast<uint16_t>(img.get_width());
update.Height = static_cast<uint16_t>(img.get_height());
update.Pixels.resize(static_cast<size_t>(update.Width) * update.Height);
for(uint32_t y = 0; y < update.Height; ++y) {
const auto& row = img.get_pixbuf().operator[](y);
for(uint32_t x = 0; x < update.Width; ++x) {
const auto& px = row[x];
uint32_t rgba = (uint32_t(px.alpha) << 24)
| (uint32_t(px.red) << 16)
| (uint32_t(px.green) << 8)
| uint32_t(px.blue);
update.Pixels[x + y * update.Width] = rgba;
}
}
}
RU.Textures.push_back(std::move(update));
updated++;
} else if(type == EnumAssets::Particle) {
RU.Particles.push_back({id, std::move(data)});
updated++;
} else if(type == EnumAssets::Animation) {
RU.Animations.push_back({id, std::move(data)});
updated++;
} else if(type == EnumAssets::Sound) {
RU.Sounds.push_back({id, std::move(data)});
updated++;
} else if(type == EnumAssets::Font) {
RU.Fonts.push_back({id, std::move(data)});
updated++;
}
}
if(updated || missingSource || missingData) {
LOG.debug() << "HashLoad type=" << int(typeIndex)
<< " updated=" << updated
<< " missingSource=" << missingSource
<< " missingData=" << missingData;
}
pending = std::move(stillPending);
}
for(const auto& [hash, res] : files)
WaitingHashes.erase(hash);

138
Src/Client/ServerSession.cpp
View File

@@ -310,25 +310,27 @@ void ServerSession::update(GlobalTime gTime, float dTime) {
// Если AssetsManager запрашивает ресурсы с сервера
{
std::vector<Hash_t> needRequest = AM.pullNeededResources();
Net::Packet pack;
std::vector<Net::Packet> packets;
if(!needRequest.empty()) {
Net::Packet pack;
std::vector<Net::Packet> packets;
auto check = [&]() {
if(pack.size() > 64000)
auto check = [&]() {
if(pack.size() > 64000)
packets.emplace_back(std::move(pack));
};
pack << (uint8_t) ToServer::L1::System << (uint8_t) ToServer::L2System::ResourceRequest;
pack << (uint16_t) needRequest.size();
for(const Hash_t& hash : needRequest) {
pack.write((const std::byte*) hash.data(), 32);
check();
}
if(pack.size())
packets.emplace_back(std::move(pack));
};
pack << (uint8_t) ToServer::L1::System << (uint8_t) ToServer::L2System::ResourceRequest;
pack << (uint16_t) needRequest.size();
for(const Hash_t& hash : needRequest) {
pack.write((const std::byte*) hash.data(), 32);
check();
Socket->pushPackets(&packets);
}
if(pack.size())
packets.emplace_back(std::move(pack));
Socket->pushPackets(&packets);
}
if(!AsyncContext.TickSequence.get_read().empty()) {
@@ -489,10 +491,10 @@ void ServerSession::update(GlobalTime gTime, float dTime) {
}
{
for(auto& [id, info] : data.Entity_AddOrChange) {
auto iter = std::lower_bound(entity_Lost.begin(), entity_Lost.end(), id);
if(iter != entity_Lost.end() && *iter == id)
entity_Lost.erase(iter);
for(auto& [id, info] : data.Entity_AddOrChange) {
auto iter = std::lower_bound(entity_Lost.begin(), entity_Lost.end(), id);
if(iter != entity_Lost.end() && *iter == id)
entity_Lost.erase(iter);
entity_AddOrChange[id] = info;
}
@@ -504,13 +506,101 @@ void ServerSession::update(GlobalTime gTime, float dTime) {
entity_Lost.insert(entity_Lost.end(), data.Entity_Lost.begin(), data.Entity_Lost.end());
std::sort(entity_Lost.begin(), entity_Lost.end());
auto eraseIter = std::unique(entity_Lost.begin(), entity_Lost.end());
entity_Lost.erase(eraseIter, entity_Lost.end());
entity_Lost.erase(eraseIter, entity_Lost.end());
}
}
{
AssetsManager::ResourceUpdates updates = AM.pullResourceUpdates();
if(!updates.Models.empty()) {
auto& map = Assets[EnumAssets::Model];
for(auto& update : updates.Models) {
AssetEntry entry;
entry.Id = update.Id;
entry.Model = std::move(update.Model);
entry.ModelHeader = std::move(update.Header);
map[entry.Id] = std::move(entry);
result.Assets_ChangeOrAdd[EnumAssets::Model].push_back(update.Id);
}
}
for(auto& [id, _] : profile_Voxel_AddOrChange)
result.Profiles_ChangeOrAdd[EnumDefContent::Voxel].push_back(id);
result.Profiles_Lost[EnumDefContent::Voxel] = profile_Voxel_Lost;
if(!updates.Nodestates.empty()) {
auto& map = Assets[EnumAssets::Nodestate];
for(auto& update : updates.Nodestates) {
AssetEntry entry;
entry.Id = update.Id;
entry.Nodestate = std::move(update.Nodestate);
entry.NodestateHeader = std::move(update.Header);
map[entry.Id] = std::move(entry);
result.Assets_ChangeOrAdd[EnumAssets::Nodestate].push_back(update.Id);
}
}
if(!updates.Textures.empty()) {
auto& map = Assets[EnumAssets::Texture];
for(auto& update : updates.Textures) {
AssetEntry entry;
entry.Id = update.Id;
entry.Domain = std::move(update.Domain);
entry.Key = std::move(update.Key);
entry.Width = update.Width;
entry.Height = update.Height;
entry.Pixels = std::move(update.Pixels);
entry.Header = std::move(update.Header);
map[entry.Id] = std::move(entry);
result.Assets_ChangeOrAdd[EnumAssets::Texture].push_back(update.Id);
}
}
if(!updates.Particles.empty()) {
auto& map = Assets[EnumAssets::Particle];
for(auto& update : updates.Particles) {
AssetEntry entry;
entry.Id = update.Id;
entry.Data = std::move(update.Data);
map[entry.Id] = std::move(entry);
result.Assets_ChangeOrAdd[EnumAssets::Particle].push_back(update.Id);
}
}
if(!updates.Animations.empty()) {
auto& map = Assets[EnumAssets::Animation];
for(auto& update : updates.Animations) {
AssetEntry entry;
entry.Id = update.Id;
entry.Data = std::move(update.Data);
map[entry.Id] = std::move(entry);
result.Assets_ChangeOrAdd[EnumAssets::Animation].push_back(update.Id);
}
}
if(!updates.Sounds.empty()) {
auto& map = Assets[EnumAssets::Sound];
for(auto& update : updates.Sounds) {
AssetEntry entry;
entry.Id = update.Id;
entry.Data = std::move(update.Data);
map[entry.Id] = std::move(entry);
result.Assets_ChangeOrAdd[EnumAssets::Sound].push_back(update.Id);
}
}
if(!updates.Fonts.empty()) {
auto& map = Assets[EnumAssets::Font];
for(auto& update : updates.Fonts) {
AssetEntry entry;
entry.Id = update.Id;
entry.Data = std::move(update.Data);
map[entry.Id] = std::move(entry);
result.Assets_ChangeOrAdd[EnumAssets::Font].push_back(update.Id);
}
}
}
for(auto& [id, _] : profile_Voxel_AddOrChange)
result.Profiles_ChangeOrAdd[EnumDefContent::Voxel].push_back(id);
result.Profiles_Lost[EnumDefContent::Voxel] = profile_Voxel_Lost;
for(auto& [id, _] : profile_Node_AddOrChange)
result.Profiles_ChangeOrAdd[EnumDefContent::Node].push_back(id);
@@ -759,6 +849,8 @@ void ServerSession::update(GlobalTime gTime, float dTime) {
RS->tickSync(result);
}
AM.tick();
// Здесь нужно обработать управляющие пакеты

12
Src/Client/Vulkan/VulkanRenderSession.cpp
View File

@@ -1680,7 +1680,7 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) {
if(auto iter = data.Profiles_Lost.find(EnumDefContent::Voxel); iter != data.Profiles_Lost.end())
mcpData.ChangedVoxels.insert(mcpData.ChangedVoxels.end(), iter->second.begin(), iter->second.end());
std::vector<std::tuple<AssetsModel, Resource, const std::vector<uint8_t>*>> modelResources;
std::vector<const AssetEntry*> modelResources;
std::vector<AssetsModel> modelLost;
if(auto iter = data.Assets_ChangeOrAdd.find(EnumAssets::Model); iter != data.Assets_ChangeOrAdd.end()) {
const auto& list = ServerSession->Assets[EnumAssets::Model];
@@ -1689,7 +1689,7 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) {
if(entryIter == list.end())
continue;
modelResources.emplace_back(id, entryIter->second.Res, &entryIter->second.Dependencies);
modelResources.push_back(&entryIter->second);
}
}
if(auto iter = data.Assets_Lost.find(EnumAssets::Model); iter != data.Assets_Lost.end())
@@ -1714,7 +1714,9 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) {
textureResources.push_back({
.Id = id,
.Res = entryIter->second.Res,
.Width = entryIter->second.Width,
.Height = entryIter->second.Height,
.Pixels = entryIter->second.Pixels,
.Domain = entryIter->second.Domain,
.Key = entryIter->second.Key
});
@@ -1730,7 +1732,7 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) {
std::vector<AssetsNodestate> changedNodestates;
if(NSP) {
std::vector<std::tuple<AssetsNodestate, Resource, const std::vector<uint8_t>*>> nodestateResources;
std::vector<const AssetEntry*> nodestateResources;
std::vector<AssetsNodestate> nodestateLost;
if(auto iter = data.Assets_ChangeOrAdd.find(EnumAssets::Nodestate); iter != data.Assets_ChangeOrAdd.end()) {
@@ -1740,7 +1742,7 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) {
if(entryIter == list.end())
continue;
nodestateResources.emplace_back(id, entryIter->second.Res, &entryIter->second.Dependencies);
nodestateResources.push_back(&entryIter->second);
}
}

230
Src/Client/Vulkan/VulkanRenderSession.hpp
View File

@@ -89,10 +89,11 @@ public:
}
// Применяет изменения, возвращая все затронутые модели
std::vector<AssetsModel> onModelChanges(std::vector<std::tuple<AssetsModel, Resource, const std::vector<uint8_t>*>> newOrChanged,
std::vector<AssetsModel> onModelChanges(std::vector<const AssetEntry*> newOrChanged,
std::vector<AssetsModel> lost,
const std::unordered_map<ResourceId, AssetEntry>* modelAssets) {
std::vector<AssetsModel> result;
(void)modelAssets;
std::move_only_function<void(ResourceId)> makeUnready;
makeUnready = [&](ResourceId id) {
@@ -136,87 +137,52 @@ public:
Models.erase(iterModel);
}
for(const auto& [key, resource, deps] : newOrChanged) {
for(const AssetEntry* entry : newOrChanged) {
if(!entry)
continue;
const AssetsModel key = entry->Id;
result.push_back(key);
makeUnready(key);
ModelObject model;
std::string type = "unknown";
std::optional<AssetsManager::ParsedHeader> header;
if(deps && !deps->empty())
header = AssetsManager::parseHeader(EnumAssets::Model, *deps);
const std::vector<uint32_t>* textureDeps = header ? &header->TextureDeps : nullptr;
auto remapTextureId = [&](uint32_t placeholder) -> uint32_t {
if(!textureDeps || placeholder >= textureDeps->size())
return 0;
return (*textureDeps)[placeholder];
};
auto remapPipeline = [&](TexturePipeline pipe) {
if(textureDeps) {
for(auto& texId : pipe.BinTextures)
texId = remapTextureId(texId);
if(!pipe.Pipeline.empty()) {
std::vector<uint8_t> code;
code.resize(pipe.Pipeline.size());
std::memcpy(code.data(), pipe.Pipeline.data(), code.size());
TexturePipelineProgram::remapTexIds(code, *textureDeps, nullptr);
pipe.Pipeline.resize(code.size());
std::memcpy(pipe.Pipeline.data(), code.data(), code.size());
}
}
return pipe;
};
const HeadlessModel& hm = entry->Model;
const HeadlessModel::Header& header = entry->ModelHeader;
size_t dataSize = 0;
std::array<uint8_t, 4> prefix = {};
try {
std::u8string_view data((const char8_t*) resource.data(), resource.size());
dataSize = data.size();
if(!data.empty()) {
const size_t prefixLen = std::min<size_t>(prefix.size(), data.size());
for(size_t i = 0; i < prefixLen; ++i)
prefix[i] = static_cast<uint8_t>(data[i]);
}
if(data.starts_with((const char8_t*) "bm")) {
type = "InternalBinary";
// Компилированная модель внутреннего формата
HeadlessModel hm;
hm.load(data);
model.TextureMap.clear();
model.TextureMap.reserve(hm.Textures.size());
for(const auto& [tkey, id] : hm.Textures) {
TexturePipeline pipe;
if(header && id < header->TexturePipelines.size()) {
pipe.Pipeline = header->TexturePipelines[id];
} else {
LOG.warn() << "Model texture pipeline id out of range: model=" << key
<< " local=" << id
<< " pipelines=" << (header ? header->TexturePipelines.size() : 0);
pipe.BinTextures.push_back(id);
pipe = remapPipeline(std::move(pipe));
}
model.TextureMap.emplace(tkey, std::move(pipe));
model.TextureMap.clear();
model.TextureMap.reserve(hm.Textures.size());
for(const auto& [tkey, id] : hm.Textures) {
TexturePipeline pipe;
if(id < header.TexturePipelines.size()) {
pipe.Pipeline = header.TexturePipelines[id];
} else {
LOG.warn() << "Model texture pipeline id out of range: model=" << key
<< " local=" << id
<< " pipelines=" << header.TexturePipelines.size();
pipe.BinTextures.push_back(id);
}
model.TextureKeys = {};
model.TextureMap.emplace(tkey, std::move(pipe));
}
model.TextureKeys = {};
for(const HeadlessModel::Cuboid& cb : hm.Cuboids) {
glm::vec3 min = glm::min(cb.From, cb.To), max = glm::max(cb.From, cb.To);
for(const HeadlessModel::Cuboid& cb : hm.Cuboids) {
glm::vec3 min = glm::min(cb.From, cb.To), max = glm::max(cb.From, cb.To);
for(const auto& [face, params] : cb.Faces) {
glm::vec2 from_uv = {params.UV[0], params.UV[1]}, to_uv = {params.UV[2], params.UV[3]};
for(const auto& [face, params] : cb.Faces) {
glm::vec2 from_uv = {params.UV[0], params.UV[1]}, to_uv = {params.UV[2], params.UV[3]};
uint32_t texId;
{
auto iter = std::find(model.TextureKeys.begin(), model.TextureKeys.end(), params.Texture);
if(iter == model.TextureKeys.end()) {
texId = model.TextureKeys.size();
model.TextureKeys.push_back(params.Texture);
} else {
texId = iter-model.TextureKeys.begin();
}
uint32_t texId;
{
auto iter = std::find(model.TextureKeys.begin(), model.TextureKeys.end(), params.Texture);
if(iter == model.TextureKeys.end()) {
texId = model.TextureKeys.size();
model.TextureKeys.push_back(params.Texture);
} else {
texId = iter-model.TextureKeys.begin();
}
}
std::vector<Vertex> v;
std::vector<Vertex> v;
auto addQuad = [&](const glm::vec3& p0,
const glm::vec3& p1,
@@ -277,20 +243,20 @@ public:
cb.Trs.apply(v);
model.Vertecies[params.Cullface].append_range(v);
}
}
}
if(!hm.SubModels.empty()) {
model.Depends.reserve(hm.SubModels.size());
for(const auto& sub : hm.SubModels) {
if(!header || sub.Id >= header->ModelDeps.size()) {
LOG.warn() << "Model sub-model id out of range: model=" << key
<< " local=" << sub.Id
<< " deps=" << (header ? header->ModelDeps.size() : 0);
continue;
}
model.Depends.emplace_back(header->ModelDeps[sub.Id], Transformations{});
if(!hm.SubModels.empty()) {
model.Depends.reserve(hm.SubModels.size());
for(const auto& sub : hm.SubModels) {
if(sub.Id >= header.Models.size()) {
LOG.warn() << "Model sub-model id out of range: model=" << key
<< " local=" << sub.Id
<< " deps=" << header.Models.size();
continue;
}
model.Depends.emplace_back(header.Models[sub.Id], Transformations{});
}
}
// struct Face {
// int TintIndex = -1;
@@ -299,18 +265,8 @@ public:
// std::vector<Transformation> Transformations;
} else if(data.starts_with((const char8_t*) "glTF")) {
type = "glb";
} else if(data.starts_with((const char8_t*) "bgl")) {
type = "InternalGLTF";
} else if(data.starts_with((const char8_t*) "{")) {
type = "InternalJson или glTF";
// Модель внутреннего формата или glTF
}
} catch(const std::exception& exc) {
LOG.warn() << "Не удалось распарсить модель " << type << ":\n\t" << exc.what();
LOG.warn() << "Не удалось собрать модель:\n\t" << exc.what();
continue;
}
@@ -321,11 +277,10 @@ public:
size_t vertexCount = 0;
for(const auto& [_, verts] : model.Vertecies)
vertexCount += verts.size();
size_t texDepsCount = textureDeps ? textureDeps->size() : 0;
LOG.debug() << "Model loaded id=" << key
<< " verts=" << vertexCount
<< " texKeys=" << model.TextureKeys.size()
<< " texDeps=" << texDepsCount;
<< " pipelines=" << header.TexturePipelines.size();
}
}
@@ -334,12 +289,7 @@ public:
uint32_t idx = debugEmptyModelLogCount.fetch_add(1);
if(idx < 128) {
LOG.warn() << "Model has empty geometry id=" << key
<< " type=" << type
<< " size=" << dataSize
<< " prefix=" << int(prefix[0]) << '.'
<< int(prefix[1]) << '.'
<< int(prefix[2]) << '.'
<< int(prefix[3]);
<< " pipelines=" << header.TexturePipelines.size();
}
}
@@ -495,7 +445,9 @@ class TextureProvider {
public:
struct TextureUpdate {
AssetsTexture Id = 0;
Resource Res;
uint16_t Width = 0;
uint16_t Height = 0;
std::vector<uint32_t> Pixels;
std::string Domain;
std::string Key;
};
@@ -639,40 +591,21 @@ public:
std::lock_guard lock(Mutex);
std::vector<AssetsTexture> result;
for(const auto& update : newOrChanged) {
for(auto& update : newOrChanged) {
const AssetsTexture key = update.Id;
const Resource& res = update.Res;
result.push_back(key);
iResource sres((const uint8_t*) res.data(), res.size());
iBinaryStream stream = sres.makeStream();
png::image<png::rgba_pixel> img(stream.Stream);
uint32_t width = img.get_width();
uint32_t height = img.get_height();
std::vector<uint32_t> pixels;
pixels.resize(width*height);
for(uint32_t y = 0; y < height; y++) {
const auto& row = img.get_pixbuf().operator [](y);
for(uint32_t x = 0; x < width; x++) {
const auto& px = row[x];
uint32_t rgba = (uint32_t(px.alpha) << 24)
| (uint32_t(px.red) << 16)
| (uint32_t(px.green) << 8)
| uint32_t(px.blue);
pixels[x + y * width] = rgba;
}
}
if(update.Width == 0 || update.Height == 0 || update.Pixels.empty())
continue;
Atlas->updateTexture(key, StoredTexture(
static_cast<uint16_t>(width),
static_cast<uint16_t>(height),
std::move(pixels)
update.Width,
update.Height,
std::move(update.Pixels)
));
bool animated = false;
if(auto anim = getDefaultAnimation(update.Key, width, height)) {
if(auto anim = getDefaultAnimation(update.Key, update.Width, update.Height)) {
AnimatedSources[key] = *anim;
animated = true;
} else {
@@ -686,7 +619,7 @@ public:
if(idx < 128) {
LOG.debug() << "Texture loaded id=" << key
<< " key=" << update.Domain << ':' << update.Key
<< " size=" << width << 'x' << height
<< " size=" << update.Width << 'x' << update.Height
<< " animated=" << (animated ? 1 : 0);
}
}
@@ -911,7 +844,7 @@ public:
{}
// Применяет изменения, возвращает изменённые описания состояний
std::vector<AssetsNodestate> onNodestateChanges(std::vector<std::tuple<AssetsNodestate, Resource, const std::vector<uint8_t>*>> newOrChanged, std::vector<AssetsNodestate> lost, std::vector<AssetsModel> changedModels) {
std::vector<AssetsNodestate> onNodestateChanges(std::vector<const AssetEntry*> newOrChanged, std::vector<AssetsNodestate> lost, std::vector<AssetsModel> changedModels) {
std::vector<AssetsNodestate> result;
for(ResourceId lostId : lost) {
@@ -923,42 +856,15 @@ public:
Nodestates.erase(iterNodestate);
}
for(const auto& [key, resource, deps] : newOrChanged) {
for(const AssetEntry* entry : newOrChanged) {
if(!entry)
continue;
const AssetsNodestate key = entry->Id;
result.push_back(key);
PreparedNodeState nodestate;
std::string type = "unknown";
try {
std::u8string_view data((const char8_t*) resource.data(), resource.size());
if(data.starts_with((const char8_t*) "bn")) {
type = "InternalBinary";
// Компилированный nodestate внутреннего формата
nodestate = PreparedNodeState(data);
} else if(data.starts_with((const char8_t*) "{")) {
type = "InternalJson";
// nodestate в json формате
} else {
type = "InternalBinaryLegacy";
// Старый двоичный формат без заголовка "bn"
std::u8string patched;
patched.reserve(data.size() + 2);
patched.push_back(u8'b');
patched.push_back(u8'n');
patched.append(data);
nodestate = PreparedNodeState(patched);
}
} catch(const std::exception& exc) {
LOG.warn() << "Не удалось распарсить nodestate " << type << ":\n\t" << exc.what();
continue;
}
if(deps && !deps->empty()) {
auto header = AssetsManager::parseHeader(EnumAssets::Nodestate, *deps);
if(header && header->Type == EnumAssets::Nodestate) {
nodestate.LocalToModel.assign(header->ModelDeps.begin(), header->ModelDeps.end());
}
}
static_cast<HeadlessNodeState&>(nodestate) = entry->Nodestate;
nodestate.LocalToModel.assign(entry->NodestateHeader.Models.begin(), entry->NodestateHeader.Models.end());
Nodestates.insert_or_assign(key, std::move(nodestate));
if(key < 64) {

161
Src/Common/Abstract.cpp
View File

@@ -828,19 +828,117 @@ Hash_t ResourceFile::calcHash(const char8_t* data, size_t size) {
return sha2::sha256((const uint8_t*) data, size);
}
uint16_t HeadlessNodeState::Header::addModel(AssetsModel id) {
auto iter = std::find(Models.begin(), Models.end(), id);
if(iter == Models.end()) {
Models.push_back(id);
return Models.size() - 1;
}
return iter - Models.begin();
}
void HeadlessNodeState::Header::load(std::u8string_view data) {
Models.clear();
if(data.empty())
return;
if(data.size() % sizeof(AssetsModel) != 0)
MAKE_ERROR("Invalid nodestate header size");
const size_t count = data.size() / sizeof(AssetsModel);
Models.resize(count);
std::memcpy(Models.data(), data.data(), data.size());
}
ResourceHeader HeadlessNodeState::Header::dump() const {
ResourceHeader rh;
rh.reserve(Models.size() * sizeof(AssetsModel));
for(AssetsModel id : Models) {
rh += std::u8string_view((const char8_t*) &id, sizeof(AssetsModel));
}
return rh;
}
uint16_t HeadlessModel::Header::addModel(AssetsModel id) {
auto iter = std::find(Models.begin(), Models.end(), id);
if(iter == Models.end()) {
Models.push_back(id);
return Models.size() - 1;
}
return iter - Models.begin();
}
uint16_t HeadlessModel::Header::addTexturePipeline(std::vector<uint8_t> pipeline) {
TexturePipelines.push_back(std::move(pipeline));
return TexturePipelines.size() - 1;
}
void HeadlessModel::Header::load(std::u8string_view data) {
Models.clear();
TexturePipelines.clear();
if(data.empty())
return;
try {
TOS::ByteBuffer buffer(data.size(), reinterpret_cast<const uint8_t*>(data.data()));
auto reader = buffer.reader();
uint16_t modelCount = reader.readUInt16();
Models.reserve(modelCount);
for(uint16_t i = 0; i < modelCount; ++i)
Models.push_back(reader.readUInt32());
uint16_t texCount = reader.readUInt16();
TexturePipelines.reserve(texCount);
for(uint16_t i = 0; i < texCount; ++i) {
uint32_t size32 = reader.readUInt32();
TOS::ByteBuffer pipe;
reader.readBuffer(pipe);
if(pipe.size() != size32)
MAKE_ERROR("Invalid model header size");
TexturePipelines.emplace_back(pipe.begin(), pipe.end());
}
} catch(const std::exception&) {
MAKE_ERROR("Invalid model header");
}
}
ResourceHeader HeadlessModel::Header::dump() const {
TOS::ByteBuffer rh;
{
uint32_t fullSize = 0;
for(const auto& vector : TexturePipelines)
fullSize += vector.size();
rh.reserve(2 + Models.size() * sizeof(AssetsModel) + 2 + 4 * TexturePipelines.size() + fullSize);
}
TOS::ByteBuffer::Writer wr;
wr << uint16_t(Models.size());
for(AssetsModel id : Models)
wr << id;
wr << uint16_t(TexturePipelines.size());
for(const auto& pipe : TexturePipelines) {
wr << uint32_t(pipe.size());
wr << pipe;
}
TOS::ByteBuffer buff = wr.complite();
return std::u8string((const char8_t*) buff.data(), buff.size());
}
ResourceHeader HeadlessNodeState::parse(const js::object& profile, const std::function<AssetsModel(const std::string_view model)>& modelResolver) {
std::vector<AssetsModel> headerIds;
Header header;
std::function<uint16_t(const std::string_view model)> headerResolver =
[&](const std::string_view model) -> uint16_t {
AssetsModel id = modelResolver(model);
auto iter = std::find(headerIds.begin(), headerIds.end(), id);
if(iter == headerIds.end()) {
headerIds.push_back(id);
return headerIds.size()-1;
}
return iter-headerIds.begin();
return header.addModel(id);
};
for(auto& [condition, variability] : profile) {
@@ -869,14 +967,7 @@ ResourceHeader HeadlessNodeState::parse(const js::object& profile, const std::fu
Routes.emplace_back(node, std::move(models));
}
ResourceHeader rh;
rh.reserve(headerIds.size()*sizeof(AssetsModel));
for(AssetsModel id : headerIds) {
rh += std::u8string_view((const char8_t*) &id, sizeof(AssetsModel));
}
return rh;
return header.dump();
}
ResourceHeader HeadlessNodeState::parse(const sol::table& profile, const std::function<AssetsModel(const std::string_view model)>& modelResolver) {
@@ -1494,21 +1585,14 @@ ResourceHeader HeadlessModel::parse(
const std::function<AssetsModel(const std::string_view model)>& modelResolver,
const std::function<std::vector<uint8_t>(const std::string_view texturePipelineSrc)>& textureResolver
) {
std::vector<AssetsModel> headerIdsModels;
Header header;
std::function<uint16_t(const std::string_view model)> headerResolverModel =
[&](const std::string_view model) -> uint16_t {
AssetsModel id = modelResolver(model);
auto iter = std::find(headerIdsModels.begin(), headerIdsModels.end(), id);
if(iter == headerIdsModels.end()) {
headerIdsModels.push_back(id);
return headerIdsModels.size()-1;
}
return iter-headerIdsModels.begin();
return header.addModel(id);
};
std::vector<std::vector<uint8_t>> headerIdsTextures;
std::unordered_map<std::string, uint32_t, detail::TSVHash, detail::TSVEq> textureToLocal;
std::function<uint16_t(const std::string_view texturePipelineSrc)> headerResolverTexture =
@@ -1518,9 +1602,8 @@ ResourceHeader HeadlessModel::parse(
return iter->second;
}
std::vector<uint8_t> program = textureResolver(texturePipelineSrc);
headerIdsTextures.push_back(program);
uint16_t id = textureToLocal[(std::string) texturePipelineSrc] = headerIdsTextures.size()-1;
uint16_t id = header.addTexturePipeline(textureResolver(texturePipelineSrc));
textureToLocal[(std::string) texturePipelineSrc] = id;
return id;
};
@@ -1748,27 +1831,7 @@ ResourceHeader HeadlessModel::parse(
// Заголовок
TOS::ByteBuffer rh;
{
uint32_t fullSize = 0;
for(const auto& vector : headerIdsTextures)
fullSize += vector.size();
rh.reserve(2+headerIdsModels.size()*sizeof(AssetsModel)+2+(4)*headerIdsTextures.size()+fullSize);
}
TOS::ByteBuffer::Writer wr;
wr << uint16_t(headerIdsModels.size());
for(AssetsModel id : headerIdsModels)
wr << id;
wr << uint16_t(headerIdsTextures.size());
for(const auto& pipe : headerIdsTextures) {
wr << uint32_t(pipe.size());
wr << pipe;
}
TOS::ByteBuffer buff = wr.complite();
return std::u8string((const char8_t*) buff.data(), buff.size());
return header.dump();
}
ResourceHeader HeadlessModel::parse(

18
Src/Common/Abstract.hpp
View File

@@ -675,6 +675,14 @@ struct HeadlessNodeState {
std::vector<Transformation> Transforms;
};
struct Header {
std::vector<AssetsModel> Models;
uint16_t addModel(AssetsModel id);
void load(std::u8string_view data);
ResourceHeader dump() const;
};
// Ноды выражений
std::vector<Node> Nodes;
// Условия -> вариации модели + веса
@@ -912,6 +920,16 @@ struct HeadlessModel {
std::optional<EnumGuiLight> GuiLight = EnumGuiLight::Default;
std::optional<bool> AmbientOcclusion = false;
struct Header {
std::vector<AssetsModel> Models;
std::vector<std::vector<uint8_t>> TexturePipelines;
uint16_t addModel(AssetsModel id);
uint16_t addTexturePipeline(std::vector<uint8_t> pipeline);
void load(std::u8string_view data);
ResourceHeader dump() const;
};
struct FullTransformation {
glm::vec3
Rotation = glm::vec3(0),