Compare commits

..

20 Commits

Author SHA1 Message Date
0b8326e278 Сохранение мира на фс 2026-01-28 23:05:27 +03:00
07ccd4dd68 Уменьшение размер промежуточного буфера. Отчистка меша чанка, если тот пуст. 2026-01-28 22:46:35 +03:00
da673b0965 Использование профилей по умолчанию для потерянных на стороне клиента 2026-01-18 09:07:25 +03:00
3fb06080db Синхронизация профилей контента 2026-01-17 18:59:36 +03:00
affdc75ebd Добавлена задержка перед выгрузкой чанков 2026-01-16 01:06:55 +06:00
49c4d77c59 Доработка менеджера контента на стороне сервера под новый провайдер идентификаторов 2026-01-16 01:05:20 +06:00
16a0fa5f7a Синхронный IdProvider 2026-01-11 22:28:03 +06:00
a29e772f35 Более нормализованная обработка ресурсов на клиенте 2026-01-07 13:56:10 +06:00
5135aa30a7 codex-5.2: тест новой версии менеджера ассетов 2026-01-07 04:03:17 +06:00
523f9725c0 Переработка менеджера ресурсов на стороне клиентов 2026-01-07 01:58:15 +06:00
c13ad06ba9 Рефакторинг кода работы с ресурсами игры на стороне сервера 2026-01-06 18:21:25 +06:00
83530a6c15 codex-5.2: новая модель объектов взаимодействия с сетью 2026-01-06 18:20:58 +06:00
b61cc9fb03 Разделение TexPipe на cpp и hpp 2026-01-06 18:20:31 +06:00
7224499d14 Документация по текстурным программам 2026-01-05 14:00:31 +06:00
ad4b0d593a Анимация текстур по сетке 2026-01-05 13:47:53 +06:00
d3add82c55 У TexPipe убрано ключевое слово tex 2026-01-05 13:46:55 +06:00
2b2be796e9 codex-5.2: Отладка сети со стороны клиента 2026-01-05 02:25:51 +06:00
6c7a6df8f6 codex-5.2: перестройка Client/AssetsManager 2026-01-05 01:50:17 +06:00
8ce820569a codex-5.2: кое как доведено до почти рабочего состояния 2026-01-05 00:35:52 +06:00
5904fe6853 codex-5.2: отладка передачи ресурсов (assets) 2026-01-04 22:30:06 +06:00
66 changed files with 8078 additions and 6126 deletions

View File

@@ -84,6 +84,16 @@ FetchContent_Declare(
FetchContent_MakeAvailable(Boost) FetchContent_MakeAvailable(Boost)
target_link_libraries(luavox_common INTERFACE Boost::asio Boost::thread Boost::json Boost::iostreams Boost::interprocess Boost::timer Boost::circular_buffer Boost::lockfree Boost::stacktrace Boost::uuid Boost::serialization Boost::nowide) target_link_libraries(luavox_common INTERFACE Boost::asio Boost::thread Boost::json Boost::iostreams Boost::interprocess Boost::timer Boost::circular_buffer Boost::lockfree Boost::stacktrace Boost::uuid Boost::serialization Boost::nowide)
# unordered_dense
FetchContent_Declare(
unordered_dense
GIT_REPOSITORY https://github.com/martinus/unordered_dense.git
GIT_TAG v4.8.1
)
FetchContent_MakeAvailable(unordered_dense)
target_link_libraries(luavox_common INTERFACE unordered_dense::unordered_dense)
# glm # glm
# find_package(glm REQUIRED) # find_package(glm REQUIRED)
# target_include_directories(${PROJECT_NAME} PUBLIC ${GLM_INCLUDE_DIR}) # target_include_directories(${PROJECT_NAME} PUBLIC ${GLM_INCLUDE_DIR})

View File

@@ -1,9 +1,13 @@
#pragma once #pragma once
#include "Common/Net.hpp"
#include <cstdint> #include <cstdint>
#include <functional>
#include <string> #include <string>
#include <string_view>
#include <unordered_map> #include <unordered_map>
#include <unordered_set> #include <unordered_set>
#include <variant>
#include <vector> #include <vector>
#include <Common/Abstract.hpp> #include <Common/Abstract.hpp>
@@ -60,15 +64,40 @@ public:
// states // states
}; };
struct AssetsModelUpdate {
ResourceId Id = 0;
HeadlessModel Model;
HeadlessModel::Header Header;
};
struct AssetsNodestateUpdate {
ResourceId Id = 0;
HeadlessNodeState Nodestate;
HeadlessNodeState::Header Header;
};
struct AssetsTextureUpdate {
ResourceId Id = 0;
uint16_t Width = 0;
uint16_t Height = 0;
std::vector<uint32_t> Pixels;
ResourceHeader Header;
};
struct AssetsBinaryUpdate {
ResourceId Id = 0;
std::u8string Data;
};
/* Интерфейс рендера текущего подключения к серверу */ /* Интерфейс рендера текущего подключения к серверу */
class IRenderSession { class IRenderSession {
public: public:
// Объект уведомления об изменениях // Объект уведомления об изменениях
struct TickSyncData { struct TickSyncData {
// Новые или изменённые используемые теперь двоичные ресурсы // Изменения в ассетах.
std::unordered_map<EnumAssets, std::vector<ResourceId>> Assets_ChangeOrAdd; std::vector<AssetsModelUpdate> AssetsModels;
// Более не используемые ресурсы std::vector<AssetsNodestateUpdate> AssetsNodestates;
std::unordered_map<EnumAssets, std::vector<ResourceId>> Assets_Lost; std::vector<AssetsTextureUpdate> AssetsTextures;
// Новые или изменённые профили контента // Новые или изменённые профили контента
std::unordered_map<EnumDefContent, std::vector<ResourceId>> Profiles_ChangeOrAdd; std::unordered_map<EnumDefContent, std::vector<ResourceId>> Profiles_ChangeOrAdd;
@@ -87,7 +116,7 @@ public:
// Началась стадия изменения данных IServerSession, все должны приостановить работу // Началась стадия изменения данных IServerSession, все должны приостановить работу
virtual void pushStageTickSync() = 0; virtual void pushStageTickSync() = 0;
// После изменения внутренних данных IServerSession, IRenderSession уведомляется об изменениях // После изменения внутренних данных IServerSession, IRenderSession уведомляется об изменениях
virtual void tickSync(const TickSyncData& data) = 0; virtual void tickSync(TickSyncData& data) = 0;
// Установить позицию для камеры // Установить позицию для камеры
virtual void setCameraPos(WorldId_t worldId, Pos::Object pos, glm::quat quat) = 0; virtual void setCameraPos(WorldId_t worldId, Pos::Object pos, glm::quat quat) = 0;
@@ -124,14 +153,6 @@ struct WorldInfo {
std::unordered_map<Pos::GlobalRegion, Region> Regions; std::unordered_map<Pos::GlobalRegion, Region> Regions;
}; };
struct VoxelInfo {
};
struct NodeInfo {
};
struct PortalInfo { struct PortalInfo {
}; };
@@ -143,28 +164,96 @@ struct EntityInfo {
glm::quat Quat = glm::quat(1.f, 0.f, 0.f, 0.f); glm::quat Quat = glm::quat(1.f, 0.f, 0.f, 0.f);
}; };
struct FuncEntityInfo { /*
Конструируются с серверными идентификаторами
*/
struct DefVoxel {
DefVoxel() = default;
DefVoxel(const std::u8string_view view) {
}
void reBind(const std::function<ResourceId(EnumAssets, ResourceId)>& am) {
}
}; };
struct DefItemInfo { struct DefNode {
std::variant<AssetsNodestate> RenderStates;
DefNode() = default;
DefNode(const std::u8string_view view) {
Net::LinearReader lr(view);
RenderStates = lr.read<uint32_t>();
}
void reBind(const std::function<ResourceId(EnumAssets, ResourceId)>& am) {
RenderStates = am(EnumAssets::Nodestate, std::get<AssetsNodestate>(RenderStates));
}
}; };
struct DefVoxel_t {}; struct DefWorld {
struct DefNode_t { DefWorld() = default;
AssetsNodestate NodestateId = 0; DefWorld(const std::u8string_view view) {
AssetsTexture TexId = 0;
}
void reBind(const std::function<ResourceId(EnumAssets, ResourceId)>& am) {
}
};
struct DefPortal {
DefPortal() = default;
DefPortal(const std::u8string_view view) {
}
void reBind(const std::function<ResourceId(EnumAssets, ResourceId)>& am) {
}
};
struct DefEntity {
DefEntity() = default;
DefEntity(const std::u8string_view view) {
}
void reBind(const std::function<ResourceId(EnumAssets, ResourceId)>& am) {
}
};
struct DefItem {
DefItem() = default;
DefItem(const std::u8string_view view) {
}
void reBind(const std::function<ResourceId(EnumAssets, ResourceId)>& am) {
}
}; };
struct AssetEntry { struct AssetEntry {
EnumAssets Type; ResourceId Id = 0;
ResourceId Id; std::string Domain;
std::string Domain, Key; std::string Key;
Resource Res;
Hash_t Hash = {}; HeadlessModel Model;
std::vector<uint8_t> Dependencies; HeadlessModel::Header ModelHeader;
HeadlessNodeState Nodestate;
HeadlessNodeState::Header NodestateHeader;
uint16_t Width = 0;
uint16_t Height = 0;
std::vector<uint32_t> Pixels;
ResourceHeader Header;
std::u8string Data;
}; };
/* /*
@@ -177,17 +266,17 @@ struct AssetEntry {
*/ */
class IServerSession { class IServerSession {
public: public:
// Используемые двоичные ресурсы // Включить логирование входящих сетевых пакетов на клиенте.
std::unordered_map<EnumAssets, std::unordered_map<ResourceId, AssetEntry>> Assets; bool DebugLogPackets = false;
// Используемые профили контента // Используемые профили контента
struct { struct {
std::unordered_map<DefVoxelId, DefVoxel_t> DefVoxel; std::unordered_map<DefVoxelId, DefVoxel> DefVoxels;
std::unordered_map<DefNodeId, DefNode_t> DefNode; std::unordered_map<DefNodeId, DefNode> DefNodes;
std::unordered_map<DefWorldId, DefWorldInfo> DefWorld; std::unordered_map<DefWorldId, DefWorld> DefWorlds;
std::unordered_map<DefPortalId, DefPortalInfo> DefPortal; std::unordered_map<DefPortalId, DefPortal> DefPortals;
std::unordered_map<DefEntityId, DefEntityInfo> DefEntity; std::unordered_map<DefEntityId, DefEntity> DefEntitys;
std::unordered_map<DefItemId, DefItemInfo> DefItem; std::unordered_map<DefItemId, DefItem> DefItems;
} Profiles; } Profiles;
// Видимый контент // Видимый контент

View File

@@ -0,0 +1,484 @@
#include "Client/AssetsHeaderCodec.hpp"
#include <cstring>
#include <unordered_set>
#include "TOSLib.hpp"
namespace LV::Client::AssetsHeaderCodec {
namespace {
struct ParsedModelHeader {
std::vector<ResourceId> ModelDeps;
std::vector<std::vector<uint8_t>> TexturePipelines;
std::vector<ResourceId> TextureDeps;
};
std::optional<std::vector<ResourceId>> parseNodestateHeaderBytes(const std::vector<uint8_t>& header) {
if(header.empty() || header.size() % sizeof(ResourceId) != 0)
return std::nullopt;
const size_t count = header.size() / sizeof(ResourceId);
std::vector<ResourceId> deps;
deps.resize(count);
for(size_t i = 0; i < count; ++i) {
ResourceId raw = 0;
std::memcpy(&raw, header.data() + i * sizeof(ResourceId), sizeof(ResourceId));
deps[i] = raw;
}
return deps;
}
struct PipelineRemapResult {
bool Ok = true;
std::string Error;
};
PipelineRemapResult remapTexturePipelineIds(std::vector<uint8_t>& code,
const std::function<uint32_t(uint32_t)>& mapId)
{
struct Range {
size_t Start = 0;
size_t End = 0;
};
enum class SrcKind : uint8_t { TexId = 0, Sub = 1 };
enum class Op : uint8_t {
End = 0,
Base_Tex = 1,
Base_Fill = 2,
Base_Anim = 3,
Resize = 10,
Transform = 11,
Opacity = 12,
NoAlpha = 13,
MakeAlpha = 14,
Invert = 15,
Brighten = 16,
Contrast = 17,
Multiply = 18,
Screen = 19,
Colorize = 20,
Anim = 21,
Overlay = 30,
Mask = 31,
LowPart = 32,
Combine = 40
};
struct SrcMeta {
SrcKind Kind = SrcKind::TexId;
uint32_t TexId = 0;
uint32_t Off = 0;
uint32_t Len = 0;
size_t TexIdOffset = 0;
};
const size_t size = code.size();
std::vector<Range> visited;
auto read8 = [&](size_t& ip, uint8_t& out)->bool{
if(ip >= size)
return false;
out = code[ip++];
return true;
};
auto read16 = [&](size_t& ip, uint16_t& out)->bool{
if(ip + 1 >= size)
return false;
out = uint16_t(code[ip]) | (uint16_t(code[ip + 1]) << 8);
ip += 2;
return true;
};
auto read24 = [&](size_t& ip, uint32_t& out)->bool{
if(ip + 2 >= size)
return false;
out = uint32_t(code[ip])
| (uint32_t(code[ip + 1]) << 8)
| (uint32_t(code[ip + 2]) << 16);
ip += 3;
return true;
};
auto read32 = [&](size_t& ip, uint32_t& out)->bool{
if(ip + 3 >= size)
return false;
out = uint32_t(code[ip])
| (uint32_t(code[ip + 1]) << 8)
| (uint32_t(code[ip + 2]) << 16)
| (uint32_t(code[ip + 3]) << 24);
ip += 4;
return true;
};
auto readSrc = [&](size_t& ip, SrcMeta& out)->bool{
uint8_t kind = 0;
if(!read8(ip, kind))
return false;
out.Kind = static_cast<SrcKind>(kind);
if(out.Kind == SrcKind::TexId) {
out.TexIdOffset = ip;
return read24(ip, out.TexId);
}
if(out.Kind == SrcKind::Sub) {
return read24(ip, out.Off) && read24(ip, out.Len);
}
return false;
};
auto patchTexId = [&](const SrcMeta& src)->PipelineRemapResult{
if(src.Kind != SrcKind::TexId)
return {};
uint32_t newId = mapId(src.TexId);
if(newId >= (1u << 24))
return {false, "TexId exceeds u24 range"};
if(src.TexIdOffset + 2 >= code.size())
return {false, "TexId patch outside pipeline"};
code[src.TexIdOffset + 0] = uint8_t(newId & 0xFFu);
code[src.TexIdOffset + 1] = uint8_t((newId >> 8) & 0xFFu);
code[src.TexIdOffset + 2] = uint8_t((newId >> 16) & 0xFFu);
return {};
};
std::function<bool(size_t, size_t)> scan;
scan = [&](size_t start, size_t end) -> bool {
if(start >= end || end > size)
return true;
for(const auto& range : visited) {
if(range.Start == start && range.End == end)
return true;
}
visited.push_back(Range{start, end});
size_t ip = start;
while(ip < end) {
uint8_t opByte = 0;
if(!read8(ip, opByte))
return false;
Op op = static_cast<Op>(opByte);
switch(op) {
case Op::End:
return true;
case Op::Base_Tex: {
SrcMeta src{};
if(!readSrc(ip, src))
return false;
PipelineRemapResult r = patchTexId(src);
if(!r.Ok)
return false;
if(src.Kind == SrcKind::Sub) {
size_t subStart = src.Off;
size_t subEnd = subStart + src.Len;
if(!scan(subStart, subEnd))
return false;
}
} break;
case Op::Base_Fill: {
uint16_t tmp16 = 0;
uint32_t tmp32 = 0;
if(!read16(ip, tmp16)) return false;
if(!read16(ip, tmp16)) return false;
if(!read32(ip, tmp32)) return false;
} break;
case Op::Base_Anim: {
SrcMeta src{};
if(!readSrc(ip, src)) return false;
PipelineRemapResult r = patchTexId(src);
if(!r.Ok) return false;
uint16_t frameW = 0;
uint16_t frameH = 0;
uint16_t frameCount = 0;
uint16_t fpsQ = 0;
uint8_t flags = 0;
if(!read16(ip, frameW)) return false;
if(!read16(ip, frameH)) return false;
if(!read16(ip, frameCount)) return false;
if(!read16(ip, fpsQ)) return false;
if(!read8(ip, flags)) return false;
(void)frameW; (void)frameH; (void)frameCount; (void)fpsQ; (void)flags;
if(src.Kind == SrcKind::Sub) {
size_t subStart = src.Off;
size_t subEnd = subStart + src.Len;
if(!scan(subStart, subEnd)) return false;
}
} break;
case Op::Resize: {
uint16_t tmp16 = 0;
if(!read16(ip, tmp16)) return false;
if(!read16(ip, tmp16)) return false;
} break;
case Op::Transform:
case Op::Opacity:
case Op::Invert:
if(!read8(ip, opByte)) return false;
break;
case Op::NoAlpha:
case Op::Brighten:
break;
case Op::MakeAlpha:
if(ip + 2 >= size) return false;
ip += 3;
break;
case Op::Contrast:
if(ip + 1 >= size) return false;
ip += 2;
break;
case Op::Multiply:
case Op::Screen: {
uint32_t tmp32 = 0;
if(!read32(ip, tmp32)) return false;
} break;
case Op::Colorize: {
uint32_t tmp32 = 0;
if(!read32(ip, tmp32)) return false;
if(!read8(ip, opByte)) return false;
} break;
case Op::Anim: {
uint16_t frameW = 0;
uint16_t frameH = 0;
uint16_t frameCount = 0;
uint16_t fpsQ = 0;
uint8_t flags = 0;
if(!read16(ip, frameW)) return false;
if(!read16(ip, frameH)) return false;
if(!read16(ip, frameCount)) return false;
if(!read16(ip, fpsQ)) return false;
if(!read8(ip, flags)) return false;
(void)frameW; (void)frameH; (void)frameCount; (void)fpsQ; (void)flags;
} break;
case Op::Overlay:
case Op::Mask: {
SrcMeta src{};
if(!readSrc(ip, src)) return false;
PipelineRemapResult r = patchTexId(src);
if(!r.Ok) return false;
if(src.Kind == SrcKind::Sub) {
size_t subStart = src.Off;
size_t subEnd = subStart + src.Len;
if(!scan(subStart, subEnd)) return false;
}
} break;
case Op::LowPart: {
if(!read8(ip, opByte)) return false;
SrcMeta src{};
if(!readSrc(ip, src)) return false;
PipelineRemapResult r = patchTexId(src);
if(!r.Ok) return false;
if(src.Kind == SrcKind::Sub) {
size_t subStart = src.Off;
size_t subEnd = subStart + src.Len;
if(!scan(subStart, subEnd)) return false;
}
} break;
case Op::Combine: {
uint16_t w = 0, h = 0, n = 0;
if(!read16(ip, w)) return false;
if(!read16(ip, h)) return false;
if(!read16(ip, n)) return false;
for(uint16_t i = 0; i < n; ++i) {
uint16_t tmp16 = 0;
if(!read16(ip, tmp16)) return false;
if(!read16(ip, tmp16)) return false;
SrcMeta src{};
if(!readSrc(ip, src)) return false;
PipelineRemapResult r = patchTexId(src);
if(!r.Ok) return false;
if(src.Kind == SrcKind::Sub) {
size_t subStart = src.Off;
size_t subEnd = subStart + src.Len;
if(!scan(subStart, subEnd)) return false;
}
}
(void)w; (void)h;
} break;
default:
return false;
}
}
return true;
};
if(!scan(0, size))
return {false, "Invalid texture pipeline bytecode"};
return {};
}
std::vector<uint32_t> collectTexturePipelineIds(const std::vector<uint8_t>& code) {
std::vector<uint32_t> out;
std::unordered_set<uint32_t> seen;
auto addId = [&](uint32_t id) {
if(seen.insert(id).second)
out.push_back(id);
};
std::vector<uint8_t> copy = code;
auto result = remapTexturePipelineIds(copy, [&](uint32_t id) {
addId(id);
return id;
});
if(!result.Ok)
return {};
return out;
}
std::optional<ParsedModelHeader> parseModelHeaderBytes(const std::vector<uint8_t>& header) {
if(header.empty())
return std::nullopt;
ParsedModelHeader result;
try {
TOS::ByteBuffer buffer(header.size(), header.data());
auto reader = buffer.reader();
uint16_t modelCount = reader.readUInt16();
result.ModelDeps.reserve(modelCount);
for(uint16_t i = 0; i < modelCount; ++i)
result.ModelDeps.push_back(reader.readUInt32());
uint16_t texCount = reader.readUInt16();
result.TexturePipelines.reserve(texCount);
for(uint16_t i = 0; i < texCount; ++i) {
uint32_t size32 = reader.readUInt32();
TOS::ByteBuffer pipe;
reader.readBuffer(pipe);
if(pipe.size() != size32)
return std::nullopt;
result.TexturePipelines.emplace_back(pipe.begin(), pipe.end());
}
std::unordered_set<ResourceId> seen;
for(const auto& pipe : result.TexturePipelines) {
for(uint32_t id : collectTexturePipelineIds(pipe)) {
if(seen.insert(id).second)
result.TextureDeps.push_back(id);
}
}
} catch(const std::exception&) {
return std::nullopt;
}
return result;
}
} // namespace
std::optional<ParsedHeader> parseHeader(EnumAssets type, const std::vector<uint8_t>& header) {
if(header.empty())
return std::nullopt;
ParsedHeader result;
result.Type = type;
if(type == EnumAssets::Nodestate) {
auto deps = parseNodestateHeaderBytes(header);
if(!deps)
return std::nullopt;
result.ModelDeps = std::move(*deps);
return result;
}
if(type == EnumAssets::Model) {
auto parsed = parseModelHeaderBytes(header);
if(!parsed)
return std::nullopt;
result.ModelDeps = std::move(parsed->ModelDeps);
result.TexturePipelines = std::move(parsed->TexturePipelines);
result.TextureDeps = std::move(parsed->TextureDeps);
return result;
}
return std::nullopt;
}
std::vector<uint8_t> rebindHeader(EnumAssets type, const std::vector<uint8_t>& header,
const MapIdFn& mapModelId, const MapIdFn& mapTextureId, const WarnFn& warn)
{
if(header.empty())
return {};
if(type == EnumAssets::Nodestate) {
if(header.size() % sizeof(ResourceId) != 0)
return header;
std::vector<uint8_t> out(header.size());
const size_t count = header.size() / sizeof(ResourceId);
for(size_t i = 0; i < count; ++i) {
ResourceId raw = 0;
std::memcpy(&raw, header.data() + i * sizeof(ResourceId), sizeof(ResourceId));
ResourceId mapped = mapModelId(raw);
std::memcpy(out.data() + i * sizeof(ResourceId), &mapped, sizeof(ResourceId));
}
return out;
}
if(type == EnumAssets::Model) {
try {
TOS::ByteBuffer buffer(header.size(), header.data());
auto reader = buffer.reader();
uint16_t modelCount = reader.readUInt16();
std::vector<ResourceId> models;
models.reserve(modelCount);
for(uint16_t i = 0; i < modelCount; ++i) {
ResourceId id = reader.readUInt32();
models.push_back(mapModelId(id));
}
uint16_t texCount = reader.readUInt16();
std::vector<std::vector<uint8_t>> pipelines;
pipelines.reserve(texCount);
for(uint16_t i = 0; i < texCount; ++i) {
uint32_t size32 = reader.readUInt32();
TOS::ByteBuffer pipe;
reader.readBuffer(pipe);
if(pipe.size() != size32) {
warn("Pipeline size mismatch");
}
std::vector<uint8_t> code(pipe.begin(), pipe.end());
auto result = remapTexturePipelineIds(code, [&](uint32_t id) {
return mapTextureId(static_cast<ResourceId>(id));
});
if(!result.Ok) {
warn(result.Error);
}
pipelines.emplace_back(std::move(code));
}
TOS::ByteBuffer::Writer wr;
wr << uint16_t(models.size());
for(ResourceId id : models)
wr << id;
wr << uint16_t(pipelines.size());
for(const auto& pipe : pipelines) {
wr << uint32_t(pipe.size());
TOS::ByteBuffer pipeBuff(pipe.begin(), pipe.end());
wr << pipeBuff;
}
TOS::ByteBuffer out = wr.complite();
return std::vector<uint8_t>(out.begin(), out.end());
} catch(const std::exception&) {
warn("Failed to rebind model header");
return header;
}
}
return header;
}
} // namespace LV::Client::AssetsHeaderCodec

View File

@@ -0,0 +1,27 @@
#pragma once
#include <cstdint>
#include <functional>
#include <optional>
#include <string>
#include <vector>
#include "Common/Abstract.hpp"
namespace LV::Client::AssetsHeaderCodec {
struct ParsedHeader {
EnumAssets Type{};
std::vector<ResourceId> ModelDeps;
std::vector<ResourceId> TextureDeps;
std::vector<std::vector<uint8_t>> TexturePipelines;
};
using MapIdFn = std::function<ResourceId(ResourceId)>;
using WarnFn = std::function<void(const std::string&)>;
std::optional<ParsedHeader> parseHeader(EnumAssets type, const std::vector<uint8_t>& header);
std::vector<uint8_t> rebindHeader(EnumAssets type, const std::vector<uint8_t>& header,
const MapIdFn& mapModelId, const MapIdFn& mapTextureId, const WarnFn& warn);
} // namespace LV::Client::AssetsHeaderCodec

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,767 @@
#include "AssetsManager.hpp"
#include <algorithm>
#include <cassert>
#include <fstream>
#include "Common/TexturePipelineProgram.hpp"
namespace LV::Client {
namespace {
static const char* assetTypeName(EnumAssets type) {
switch(type) {
case EnumAssets::Nodestate: return "nodestate";
case EnumAssets::Model: return "model";
case EnumAssets::Texture: return "texture";
case EnumAssets::Particle: return "particle";
case EnumAssets::Animation: return "animation";
case EnumAssets::Sound: return "sound";
case EnumAssets::Font: return "font";
default: return "unknown";
}
}
static const char* enumAssetsToDirectory(LV::EnumAssets value) {
switch(value) {
case LV::EnumAssets::Nodestate: return "nodestate";
case LV::EnumAssets::Particle: return "particle";
case LV::EnumAssets::Animation: return "animation";
case LV::EnumAssets::Model: return "model";
case LV::EnumAssets::Texture: return "texture";
case LV::EnumAssets::Sound: return "sound";
case LV::EnumAssets::Font: return "font";
default:
break;
}
assert(!"Unknown asset type");
return "";
}
static std::u8string readFileBytes(const fs::path& path) {
std::ifstream file(path, std::ios::binary);
if(!file)
throw std::runtime_error("Не удалось открыть файл: " + path.string());
file.seekg(0, std::ios::end);
std::streamoff size = file.tellg();
if(size < 0)
size = 0;
file.seekg(0, std::ios::beg);
std::u8string data;
data.resize(static_cast<size_t>(size));
if(size > 0) {
file.read(reinterpret_cast<char*>(data.data()), size);
if(!file)
throw std::runtime_error("Не удалось прочитать файл: " + path.string());
}
return data;
}
static std::u8string readOptionalMeta(const fs::path& path) {
fs::path metaPath = path;
metaPath += ".meta";
if(!fs::exists(metaPath) || !fs::is_regular_file(metaPath))
return {};
return readFileBytes(metaPath);
}
} // namespace
AssetsManager::AssetsManager(asio::io_context& ioc, const fs::path& cachePath,
size_t maxCacheDirectorySize, size_t maxLifeTime)
: Cache(AssetsCacheManager::Create(ioc, cachePath, maxCacheDirectorySize, maxLifeTime))
{
for(size_t i = 0; i < static_cast<size_t>(AssetType::MAX_ENUM); ++i)
Types[i].NextLocalId = 1;
initSources();
}
void AssetsManager::initSources() {
using SourceResult = AssetsManager::SourceResult;
using SourceStatus = AssetsManager::SourceStatus;
using SourceReady = AssetsManager::SourceReady;
using ResourceKey = AssetsManager::ResourceKey;
using PackResource = AssetsManager::PackResource;
class PackSource final : public IResourceSource {
public:
explicit PackSource(AssetsManager* manager) : Manager(manager) {}
SourceResult tryGet(const ResourceKey& key) override {
std::optional<PackResource> pack = Manager->findPackResource(key.Type, key.Domain, key.Key);
if(pack && pack->Hash == key.Hash)
return {SourceStatus::Hit, pack->Res, 0};
return {SourceStatus::Miss, std::nullopt, 0};
}
void collectReady(std::vector<SourceReady>&) override {}
bool isAsync() const override {
return false;
}
void startPending(std::vector<Hash_t>) override {}
private:
AssetsManager* Manager = nullptr;
};
class MemorySource final : public IResourceSource {
public:
explicit MemorySource(AssetsManager* manager) : Manager(manager) {}
SourceResult tryGet(const ResourceKey& key) override {
auto iter = Manager->MemoryResourcesByHash.find(key.Hash);
if(iter == Manager->MemoryResourcesByHash.end())
return {SourceStatus::Miss, std::nullopt, 0};
return {SourceStatus::Hit, iter->second, 0};
}
void collectReady(std::vector<SourceReady>&) override {}
bool isAsync() const override {
return false;
}
void startPending(std::vector<Hash_t>) override {}
private:
AssetsManager* Manager = nullptr;
};
class CacheSource final : public IResourceSource {
public:
CacheSource(AssetsManager* manager, size_t sourceIndex)
: Manager(manager), SourceIndex(sourceIndex) {}
SourceResult tryGet(const ResourceKey&) override {
return {SourceStatus::Pending, std::nullopt, SourceIndex};
}
void collectReady(std::vector<SourceReady>& out) override {
std::vector<std::pair<Hash_t, std::optional<Resource>>> cached = Manager->Cache->pullReads();
out.reserve(out.size() + cached.size());
for(auto& [hash, res] : cached)
out.push_back(SourceReady{hash, res, SourceIndex});
}
bool isAsync() const override {
return true;
}
void startPending(std::vector<Hash_t> hashes) override {
if(!hashes.empty())
Manager->Cache->pushReads(std::move(hashes));
}
private:
AssetsManager* Manager = nullptr;
size_t SourceIndex = 0;
};
Sources.clear();
PackSourceIndex = Sources.size();
Sources.push_back(SourceEntry{std::make_unique<PackSource>(this), 0});
MemorySourceIndex = Sources.size();
Sources.push_back(SourceEntry{std::make_unique<MemorySource>(this), 0});
CacheSourceIndex = Sources.size();
Sources.push_back(SourceEntry{std::make_unique<CacheSource>(this, CacheSourceIndex), 0});
}
void AssetsManager::collectReadyFromSources() {
std::vector<SourceReady> ready;
for(auto& entry : Sources)
entry.Source->collectReady(ready);
for(SourceReady& item : ready) {
auto iter = PendingReadsByHash.find(item.Hash);
if(iter == PendingReadsByHash.end())
continue;
if(item.Value)
registerSourceHit(item.Hash, item.SourceIndex);
for(ResourceKey& key : iter->second) {
if(item.SourceIndex == CacheSourceIndex) {
if(item.Value) {
LOG.debug() << "Cache hit type=" << assetTypeName(key.Type)
<< " id=" << key.Id
<< " key=" << key.Domain << ':' << key.Key
<< " hash=" << int(item.Hash[0]) << '.'
<< int(item.Hash[1]) << '.'
<< int(item.Hash[2]) << '.'
<< int(item.Hash[3])
<< " size=" << item.Value->size();
} else {
LOG.debug() << "Cache miss type=" << assetTypeName(key.Type)
<< " id=" << key.Id
<< " key=" << key.Domain << ':' << key.Key
<< " hash=" << int(item.Hash[0]) << '.'
<< int(item.Hash[1]) << '.'
<< int(item.Hash[2]) << '.'
<< int(item.Hash[3]);
}
}
ReadyReads.emplace_back(std::move(key), item.Value);
}
PendingReadsByHash.erase(iter);
}
}
AssetsManager::SourceResult AssetsManager::querySources(const ResourceKey& key) {
auto cacheIter = SourceCacheByHash.find(key.Hash);
if(cacheIter != SourceCacheByHash.end()) {
const size_t cachedIndex = cacheIter->second.SourceIndex;
if(cachedIndex < Sources.size()
&& cacheIter->second.Generation == Sources[cachedIndex].Generation)
{
SourceResult cached = Sources[cachedIndex].Source->tryGet(key);
cached.SourceIndex = cachedIndex;
if(cached.Status != SourceStatus::Miss)
return cached;
}
SourceCacheByHash.erase(cacheIter);
}
SourceResult pending;
pending.Status = SourceStatus::Miss;
for(size_t i = 0; i < Sources.size(); ++i) {
SourceResult res = Sources[i].Source->tryGet(key);
res.SourceIndex = i;
if(res.Status == SourceStatus::Hit) {
registerSourceHit(key.Hash, i);
return res;
}
if(res.Status == SourceStatus::Pending && pending.Status == SourceStatus::Miss)
pending = res;
}
return pending;
}
void AssetsManager::registerSourceHit(const Hash_t& hash, size_t sourceIndex) {
if(sourceIndex >= Sources.size())
return;
if(Sources[sourceIndex].Source->isAsync())
return;
SourceCacheByHash[hash] = SourceCacheEntry{
.SourceIndex = sourceIndex,
.Generation = Sources[sourceIndex].Generation
};
}
void AssetsManager::invalidateSourceCache(size_t sourceIndex) {
if(sourceIndex >= Sources.size())
return;
Sources[sourceIndex].Generation++;
for(auto iter = SourceCacheByHash.begin(); iter != SourceCacheByHash.end(); ) {
if(iter->second.SourceIndex == sourceIndex)
iter = SourceCacheByHash.erase(iter);
else
++iter;
}
}
void AssetsManager::invalidateAllSourceCache() {
for(auto& entry : Sources)
entry.Generation++;
SourceCacheByHash.clear();
}
void AssetsManager::tickSources() {
collectReadyFromSources();
}
AssetsManager::PackReloadResult AssetsManager::reloadPacks(const PackRegister& reg) {
PackReloadResult result;
std::array<PackTable, static_cast<size_t>(AssetType::MAX_ENUM)> oldPacks;
for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); ++type) {
oldPacks[type] = Types[type].PackResources;
Types[type].PackResources.clear();
}
for(const fs::path& instance : reg.Packs) {
try {
if(fs::is_regular_file(instance)) {
LOG.warn() << "Архивы ресурспаков пока не поддерживаются: " << instance.string();
continue;
}
if(!fs::is_directory(instance)) {
LOG.warn() << "Неизвестный тип ресурспака: " << instance.string();
continue;
}
fs::path assetsRoot = instance;
fs::path assetsCandidate = instance / "assets";
if(fs::exists(assetsCandidate) && fs::is_directory(assetsCandidate))
assetsRoot = assetsCandidate;
for(auto begin = fs::directory_iterator(assetsRoot), end = fs::directory_iterator(); begin != end; ++begin) {
if(!begin->is_directory())
continue;
fs::path domainPath = begin->path();
std::string domain = domainPath.filename().string();
for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); ++type) {
AssetType assetType = static_cast<AssetType>(type);
fs::path assetPath = domainPath / enumAssetsToDirectory(assetType);
if(!fs::exists(assetPath) || !fs::is_directory(assetPath))
continue;
auto& typeTable = Types[type].PackResources[domain];
for(auto fbegin = fs::recursive_directory_iterator(assetPath),
fend = fs::recursive_directory_iterator();
fbegin != fend; ++fbegin) {
if(fbegin->is_directory())
continue;
fs::path file = fbegin->path();
if(assetType == AssetType::Texture && file.extension() == ".meta")
continue;
std::string key = fs::relative(file, assetPath).generic_string();
if(typeTable.contains(key))
continue;
PackResource entry;
entry.Type = assetType;
entry.Domain = domain;
entry.Key = key;
entry.LocalId = getOrCreateLocalId(assetType, entry.Domain, entry.Key);
try {
if(assetType == AssetType::Nodestate) {
std::u8string data = readFileBytes(file);
std::string_view view(reinterpret_cast<const char*>(data.data()), data.size());
js::object obj = js::parse(view).as_object();
HeadlessNodeState hns;
auto modelResolver = [&](std::string_view model) -> AssetsModel {
auto [mDomain, mKey] = parseDomainKey(model, entry.Domain);
return getOrCreateLocalId(AssetType::Model, mDomain, mKey);
};
entry.Header = hns.parse(obj, modelResolver);
std::u8string compiled = hns.dump();
entry.Res = Resource(std::move(compiled));
entry.Hash = entry.Res.hash();
} else if(assetType == AssetType::Model) {
const std::string ext = file.extension().string();
if(ext == ".json") {
std::u8string data = readFileBytes(file);
std::string_view view(reinterpret_cast<const char*>(data.data()), data.size());
js::object obj = js::parse(view).as_object();
HeadlessModel hm;
auto modelResolver = [&](std::string_view model) -> AssetsModel {
auto [mDomain, mKey] = parseDomainKey(model, entry.Domain);
return getOrCreateLocalId(AssetType::Model, mDomain, mKey);
};
auto normalizeTexturePipelineSrc = [](std::string_view src) -> std::string {
std::string out(src);
auto isSpace = [](unsigned char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; };
size_t start = 0;
while(start < out.size() && isSpace(static_cast<unsigned char>(out[start])))
++start;
if(out.compare(start, 3, "tex") != 0) {
std::string pref = "tex ";
pref += out.substr(start);
return pref;
}
return out;
};
auto textureResolver = [&](std::string_view textureSrc) -> std::vector<uint8_t> {
TexturePipelineProgram tpp;
if(!tpp.compile(normalizeTexturePipelineSrc(textureSrc)))
return {};
auto textureIdResolver = [&](std::string_view name) -> std::optional<uint32_t> {
auto [tDomain, tKey] = parseDomainKey(name, entry.Domain);
return getOrCreateLocalId(AssetType::Texture, tDomain, tKey);
};
if(!tpp.link(textureIdResolver))
return {};
return tpp.toBytes();
};
entry.Header = hm.parse(obj, modelResolver, textureResolver);
std::u8string compiled = hm.dump();
entry.Res = Resource(std::move(compiled));
entry.Hash = entry.Res.hash();
} else {
LOG.warn() << "Не поддерживаемый формат модели: " << file.string();
continue;
}
} else if(assetType == AssetType::Texture) {
std::u8string data = readFileBytes(file);
entry.Res = Resource(std::move(data));
entry.Hash = entry.Res.hash();
entry.Header = readOptionalMeta(file);
} else {
std::u8string data = readFileBytes(file);
entry.Res = Resource(std::move(data));
entry.Hash = entry.Res.hash();
}
} catch(const std::exception& exc) {
LOG.warn() << "Ошибка загрузки ресурса " << file.string() << ": " << exc.what();
continue;
}
typeTable.emplace(entry.Key, entry);
}
}
}
} catch(const std::exception& exc) {
LOG.warn() << "Ошибка загрузки ресурспака " << instance.string() << ": " << exc.what();
}
}
for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); ++type) {
for(const auto& [domain, keyTable] : Types[type].PackResources) {
for(const auto& [key, res] : keyTable) {
bool changed = true;
auto oldDomain = oldPacks[type].find(domain);
if(oldDomain != oldPacks[type].end()) {
auto oldKey = oldDomain->second.find(key);
if(oldKey != oldDomain->second.end()) {
changed = oldKey->second.Hash != res.Hash;
}
}
if(changed)
result.ChangeOrAdd[type].push_back(res.LocalId);
}
}
for(const auto& [domain, keyTable] : oldPacks[type]) {
for(const auto& [key, res] : keyTable) {
auto newDomain = Types[type].PackResources.find(domain);
bool lost = true;
if(newDomain != Types[type].PackResources.end()) {
if(newDomain->second.contains(key))
lost = false;
}
if(lost)
result.Lost[type].push_back(res.LocalId);
}
}
}
invalidateAllSourceCache();
return result;
}
AssetsManager::BindResult AssetsManager::bindServerResource(AssetType type, AssetId serverId,
std::string domain, std::string key, const Hash_t& hash, std::vector<uint8_t> header)
{
BindResult result;
AssetId localFromDK = getOrCreateLocalId(type, domain, key);
auto& map = Types[static_cast<size_t>(type)].ServerToLocal;
AssetId localFromServer = 0;
if(serverId < map.size())
localFromServer = map[serverId];
if(localFromServer != 0)
unionLocalIds(type, localFromServer, localFromDK, &result.ReboundFrom);
AssetId localId = resolveLocalIdMutable(type, localFromDK);
if(serverId >= map.size())
map.resize(serverId + 1, 0);
map[serverId] = localId;
auto& infoList = Types[static_cast<size_t>(type)].BindInfos;
if(localId >= infoList.size())
infoList.resize(localId + 1);
bool hadBinding = infoList[localId].has_value();
bool changed = !hadBinding || infoList[localId]->Hash != hash || infoList[localId]->Header != header;
infoList[localId] = BindInfo{
.Type = type,
.LocalId = localId,
.Domain = std::move(domain),
.Key = std::move(key),
.Hash = hash,
.Header = std::move(header)
};
result.LocalId = localId;
result.Changed = changed;
result.NewBinding = !hadBinding;
return result;
}
std::optional<AssetsManager::AssetId> AssetsManager::unbindServerResource(AssetType type, AssetId serverId) {
auto& map = Types[static_cast<size_t>(type)].ServerToLocal;
if(serverId >= map.size())
return std::nullopt;
AssetId localId = map[serverId];
map[serverId] = 0;
if(localId == 0)
return std::nullopt;
return resolveLocalIdMutable(type, localId);
}
void AssetsManager::clearServerBindings() {
for(auto& typeData : Types) {
typeData.ServerToLocal.clear();
typeData.BindInfos.clear();
}
}
const AssetsManager::BindInfo* AssetsManager::getBind(AssetType type, AssetId localId) const {
localId = resolveLocalId(type, localId);
const auto& table = Types[static_cast<size_t>(type)].BindInfos;
if(localId >= table.size())
return nullptr;
if(!table[localId])
return nullptr;
return &*table[localId];
}
std::vector<uint8_t> AssetsManager::rebindHeader(AssetType type, const std::vector<uint8_t>& header, bool serverIds) {
auto mapModelId = [&](AssetId id) -> AssetId {
if(serverIds) {
auto localId = getLocalIdFromServer(AssetType::Model, id);
if(!localId) {
assert(!"Missing server bind for model id");
MAKE_ERROR("Нет бинда сервера для модели id=" << id);
}
return *localId;
}
return resolveLocalIdMutable(AssetType::Model, id);
};
auto mapTextureId = [&](AssetId id) -> AssetId {
if(serverIds) {
auto localId = getLocalIdFromServer(AssetType::Texture, id);
if(!localId) {
assert(!"Missing server bind for texture id");
MAKE_ERROR("Нет бинда сервера для текстуры id=" << id);
}
return *localId;
}
return resolveLocalIdMutable(AssetType::Texture, id);
};
auto warn = [&](const std::string& msg) {
LOG.warn() << msg;
};
return AssetsHeaderCodec::rebindHeader(type, header, mapModelId, mapTextureId, warn);
}
std::optional<AssetsManager::ParsedHeader> AssetsManager::parseHeader(AssetType type, const std::vector<uint8_t>& header) {
return AssetsHeaderCodec::parseHeader(type, header);
}
void AssetsManager::pushResources(std::vector<Resource> resources) {
for(const Resource& res : resources) {
Hash_t hash = res.hash();
MemoryResourcesByHash[hash] = res;
SourceCacheByHash.erase(hash);
registerSourceHit(hash, MemorySourceIndex);
auto iter = PendingReadsByHash.find(hash);
if(iter != PendingReadsByHash.end()) {
for(ResourceKey& key : iter->second)
ReadyReads.emplace_back(std::move(key), res);
PendingReadsByHash.erase(iter);
}
}
Cache->pushResources(std::move(resources));
}
void AssetsManager::pushReads(std::vector<ResourceKey> reads) {
std::unordered_map<size_t, std::vector<Hash_t>> pendingBySource;
for(ResourceKey& key : reads) {
SourceResult res = querySources(key);
if(res.Status == SourceStatus::Hit) {
if(res.SourceIndex == PackSourceIndex && res.Value) {
LOG.debug() << "Pack hit type=" << assetTypeName(key.Type)
<< " id=" << key.Id
<< " key=" << key.Domain << ':' << key.Key
<< " hash=" << int(key.Hash[0]) << '.'
<< int(key.Hash[1]) << '.'
<< int(key.Hash[2]) << '.'
<< int(key.Hash[3])
<< " size=" << res.Value->size();
}
ReadyReads.emplace_back(std::move(key), res.Value);
continue;
}
if(res.Status == SourceStatus::Pending) {
auto& list = PendingReadsByHash[key.Hash];
bool isFirst = list.empty();
list.push_back(std::move(key));
if(isFirst)
pendingBySource[res.SourceIndex].push_back(list.front().Hash);
continue;
}
ReadyReads.emplace_back(std::move(key), std::nullopt);
}
for(auto& [sourceIndex, hashes] : pendingBySource) {
if(sourceIndex < Sources.size())
Sources[sourceIndex].Source->startPending(std::move(hashes));
}
}
std::vector<std::pair<AssetsManager::ResourceKey, std::optional<Resource>>> AssetsManager::pullReads() {
tickSources();
std::vector<std::pair<ResourceKey, std::optional<Resource>>> out;
out.reserve(ReadyReads.size());
for(auto& entry : ReadyReads)
out.emplace_back(std::move(entry));
ReadyReads.clear();
return out;
}
AssetsManager::AssetId AssetsManager::getOrCreateLocalId(AssetType type, std::string_view domain, std::string_view key) {
auto& table = Types[static_cast<size_t>(type)].DKToLocal;
auto iterDomain = table.find(domain);
if(iterDomain == table.end()) {
iterDomain = table.emplace(
std::string(domain),
std::unordered_map<std::string, AssetId, detail::TSVHash, detail::TSVEq>{}
).first;
}
auto& keyTable = iterDomain->second;
auto iterKey = keyTable.find(key);
if(iterKey != keyTable.end()) {
iterKey->second = resolveLocalIdMutable(type, iterKey->second);
return iterKey->second;
}
AssetId id = allocateLocalId(type);
keyTable.emplace(std::string(key), id);
auto& dk = Types[static_cast<size_t>(type)].LocalToDK;
if(id >= dk.size())
dk.resize(id + 1);
dk[id] = DomainKey{std::string(domain), std::string(key), true};
return id;
}
std::optional<AssetsManager::AssetId> AssetsManager::getLocalIdFromServer(AssetType type, AssetId serverId) const {
const auto& map = Types[static_cast<size_t>(type)].ServerToLocal;
if(serverId >= map.size())
return std::nullopt;
AssetId local = map[serverId];
if(local == 0)
return std::nullopt;
return resolveLocalId(type, local);
}
AssetsManager::AssetId AssetsManager::resolveLocalId(AssetType type, AssetId localId) const {
if(localId == 0)
return 0;
const auto& parents = Types[static_cast<size_t>(type)].LocalParent;
if(localId >= parents.size())
return localId;
AssetId cur = localId;
while(cur < parents.size() && parents[cur] != cur && parents[cur] != 0)
cur = parents[cur];
return cur;
}
AssetsManager::AssetId AssetsManager::allocateLocalId(AssetType type) {
auto& next = Types[static_cast<size_t>(type)].NextLocalId;
AssetId id = next++;
auto& parents = Types[static_cast<size_t>(type)].LocalParent;
if(id >= parents.size())
parents.resize(id + 1, 0);
parents[id] = id;
auto& dk = Types[static_cast<size_t>(type)].LocalToDK;
if(id >= dk.size())
dk.resize(id + 1);
return id;
}
AssetsManager::AssetId AssetsManager::resolveLocalIdMutable(AssetType type, AssetId localId) {
if(localId == 0)
return 0;
auto& parents = Types[static_cast<size_t>(type)].LocalParent;
if(localId >= parents.size())
return localId;
AssetId root = localId;
while(root < parents.size() && parents[root] != root && parents[root] != 0)
root = parents[root];
if(root == localId)
return root;
AssetId cur = localId;
while(cur < parents.size() && parents[cur] != root && parents[cur] != 0) {
AssetId next = parents[cur];
parents[cur] = root;
cur = next;
}
return root;
}
void AssetsManager::unionLocalIds(AssetType type, AssetId fromId, AssetId toId, std::optional<AssetId>* reboundFrom) {
AssetId fromRoot = resolveLocalIdMutable(type, fromId);
AssetId toRoot = resolveLocalIdMutable(type, toId);
if(fromRoot == 0 || toRoot == 0 || fromRoot == toRoot)
return;
auto& parents = Types[static_cast<size_t>(type)].LocalParent;
if(fromRoot >= parents.size() || toRoot >= parents.size())
return;
parents[fromRoot] = toRoot;
if(reboundFrom)
*reboundFrom = fromRoot;
auto& dk = Types[static_cast<size_t>(type)].LocalToDK;
if(fromRoot < dk.size()) {
const DomainKey& fromDK = dk[fromRoot];
if(fromDK.Known) {
if(toRoot >= dk.size())
dk.resize(toRoot + 1);
DomainKey& toDK = dk[toRoot];
if(!toDK.Known) {
toDK = fromDK;
Types[static_cast<size_t>(type)].DKToLocal[toDK.Domain][toDK.Key] = toRoot;
} else if(toDK.Domain != fromDK.Domain || toDK.Key != fromDK.Key) {
LOG.warn() << "Конфликт домен/ключ при ребинде: "
<< fromDK.Domain << ':' << fromDK.Key << " vs "
<< toDK.Domain << ':' << toDK.Key;
}
}
}
auto& binds = Types[static_cast<size_t>(type)].BindInfos;
if(fromRoot < binds.size()) {
if(toRoot >= binds.size())
binds.resize(toRoot + 1);
if(!binds[toRoot] && binds[fromRoot])
binds[toRoot] = std::move(binds[fromRoot]);
}
}
std::optional<AssetsManager::PackResource> AssetsManager::findPackResource(AssetType type,
std::string_view domain, std::string_view key) const
{
const auto& typeTable = Types[static_cast<size_t>(type)].PackResources;
auto iterDomain = typeTable.find(domain);
if(iterDomain == typeTable.end())
return std::nullopt;
auto iterKey = iterDomain->second.find(key);
if(iterKey == iterDomain->second.end())
return std::nullopt;
return iterKey->second;
}
} // namespace LV::Client

View File

@@ -1,5 +1,6 @@
#pragma once #pragma once
#include <algorithm>
#include <array> #include <array>
#include <cstddef> #include <cstddef>
#include <cstdint> #include <cstdint>
@@ -9,144 +10,629 @@
#include <string> #include <string>
#include <string_view> #include <string_view>
#include <unordered_map> #include <unordered_map>
#include <unordered_set>
#include <utility> #include <utility>
#include <vector> #include <vector>
#include <cstring>
#include "Client/AssetsCacheManager.hpp" #include "Client/AssetsCacheManager.hpp"
#include "Client/AssetsHeaderCodec.hpp"
#include "Common/Abstract.hpp" #include "Common/Abstract.hpp"
#include "Common/IdProvider.hpp"
#include "Common/AssetsPreloader.hpp"
#include "Common/TexturePipelineProgram.hpp"
#include "TOSLib.hpp" #include "TOSLib.hpp"
#include "assets.hpp"
#include "boost/asio/io_context.hpp"
#include "png++/image.hpp"
#include <fstream>
#include "Abstract.hpp"
namespace LV::Client { namespace LV::Client {
namespace fs = std::filesystem; namespace fs = std::filesystem;
class AssetsManager { class AssetsManager : public IdProvider<EnumAssets> {
public: public:
using Ptr = std::shared_ptr<AssetsManager>; struct ResourceUpdates {
using AssetType = EnumAssets;
using AssetId = ResourceId;
struct ResourceKey { std::vector<AssetsModelUpdate> Models;
Hash_t Hash{}; std::vector<AssetsNodestateUpdate> Nodestates;
AssetType Type{}; std::vector<AssetsTextureUpdate> Textures;
std::string Domain; std::vector<AssetsBinaryUpdate> Particles;
std::string Key; std::vector<AssetsBinaryUpdate> Animations;
AssetId Id = 0; std::vector<AssetsBinaryUpdate> Sounds;
std::vector<AssetsBinaryUpdate> Fonts;
}; };
struct BindInfo { public:
AssetType Type{}; AssetsManager(asio::io_context& ioc, fs::path cachePath)
AssetId LocalId = 0; : Cache(AssetsCacheManager::Create(ioc, cachePath))
std::string Domain; {
std::string Key; for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); type++) {
Hash_t Hash{}; ServerToClientMap[type].push_back(0);
std::vector<uint8_t> Header; }
};
struct BindResult {
AssetId LocalId = 0;
bool Changed = false;
bool NewBinding = false;
std::optional<AssetId> ReboundFrom;
};
struct PackRegister {
std::vector<fs::path> Packs;
};
struct PackResource {
AssetType Type{};
AssetId LocalId = 0;
std::string Domain;
std::string Key;
Resource Res;
Hash_t Hash{};
std::u8string Header;
};
struct PackReloadResult {
std::array<std::vector<AssetId>, static_cast<size_t>(AssetType::MAX_ENUM)> ChangeOrAdd;
std::array<std::vector<AssetId>, static_cast<size_t>(AssetType::MAX_ENUM)> Lost;
};
struct ParsedHeader {
AssetType Type{};
std::vector<AssetId> ModelDeps;
std::vector<AssetId> TextureDeps;
std::vector<std::vector<uint8_t>> TexturePipelines;
};
static Ptr Create(asio::io_context& ioc, const fs::path& cachePath,
size_t maxCacheDirectorySize = 8 * 1024 * 1024 * 1024ULL,
size_t maxLifeTime = 7 * 24 * 60 * 60) {
return Ptr(new AssetsManager(ioc, cachePath, maxCacheDirectorySize, maxLifeTime));
} }
PackReloadResult reloadPacks(const PackRegister& reg); // Ручные обновления
struct Out_checkAndPrepareResourcesUpdate {
AssetsPreloader::Out_checkAndPrepareResourcesUpdate RP, ES;
BindResult bindServerResource(AssetType type, AssetId serverId, std::string domain, std::string key, std::unordered_map<ResourceFile::Hash_t, std::u8string> Files;
const Hash_t& hash, std::vector<uint8_t> header); };
std::optional<AssetId> unbindServerResource(AssetType type, AssetId serverId);
void clearServerBindings();
const BindInfo* getBind(AssetType type, AssetId localId) const; Out_checkAndPrepareResourcesUpdate checkAndPrepareResourcesUpdate(
const std::vector<fs::path>& resourcePacks,
const std::vector<fs::path>& extraSources
) {
Out_checkAndPrepareResourcesUpdate result;
std::vector<uint8_t> rebindHeader(AssetType type, const std::vector<uint8_t>& header, bool serverIds = true); result.RP = ResourcePacks.checkAndPrepareResourcesUpdate(
static std::optional<ParsedHeader> parseHeader(AssetType type, const std::vector<uint8_t>& header); AssetsPreloader::AssetsRegister{resourcePacks},
[&](EnumAssets type, std::string_view domain, std::string_view key) -> ResourceId {
return getId(type, domain, key);
},
[&](std::u8string&& data, ResourceFile::Hash_t hash, fs::path path) {
result.Files.emplace(hash, std::move(data));
}
);
void pushResources(std::vector<Resource> resources) { result.ES = ExtraSource.checkAndPrepareResourcesUpdate(
Cache->pushResources(std::move(resources)); AssetsPreloader::AssetsRegister{resourcePacks},
[&](EnumAssets type, std::string_view domain, std::string_view key) -> ResourceId {
return getId(type, domain, key);
}
);
return result;
} }
void pushReads(std::vector<ResourceKey> reads); struct Out_applyResourcesUpdate {
std::vector<std::pair<ResourceKey, std::optional<Resource>>> pullReads();
};
AssetId getOrCreateLocalId(AssetType type, std::string_view domain, std::string_view key); Out_applyResourcesUpdate applyResourcesUpdate(const Out_checkAndPrepareResourcesUpdate& orr) {
AssetId getOrCreateLocalFromServer(AssetType type, AssetId serverId); Out_applyResourcesUpdate result;
std::optional<AssetId> getLocalIdFromServer(AssetType type, AssetId serverId) const;
for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); ++type) {
for(ResourceId id : orr.RP.LostLinks[type]) {
std::optional<AssetsPreloader::Out_Resource> res = ResourcePacks.getResource((EnumAssets) type, id);
assert(res);
auto hashIter = HashToPath.find(res->Hash);
assert(hashIter != HashToPath.end());
auto& entry = hashIter->second;
auto iter = std::find(entry.begin(), entry.end(), res->Path);
assert(iter != entry.end());
entry.erase(iter);
if(entry.empty())
HashToPath.erase(hashIter);
}
}
ResourcePacks.applyResourcesUpdate(orr.RP);
ExtraSource.applyResourcesUpdate(orr.ES);
std::unordered_set<ResourceFile::Hash_t> needHashes;
for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); ++type) {
for(const auto& res : orr.RP.ResourceUpdates[type]) {
// Помечаем ресурс для обновления
PendingUpdateFromAsync[type].push_back(std::get<ResourceId>(res));
HashToPath[std::get<ResourceFile::Hash_t>(res)].push_back(std::get<fs::path>(res));
}
for(ResourceId id : orr.RP.LostLinks[type]) {
// Помечаем ресурс для обновления
PendingUpdateFromAsync[type].push_back(id);
auto& hh = ServerIdToHH[type];
if(id < hh.size())
needHashes.insert(std::get<ResourceFile::Hash_t>(hh[id]));
}
}
{
for(const auto& [hash, data] : orr.Files) {
WaitingHashes.insert(hash);
}
for(const auto& hash : WaitingHashes)
needHashes.erase(hash);
std::vector<std::tuple<ResourceFile::Hash_t, fs::path>> toDisk;
std::vector<ResourceFile::Hash_t> toCache;
// Теперь раскидаем хеши по доступным источникам.
for(const auto& hash : needHashes) {
auto iter = HashToPath.find(hash);
if(iter != HashToPath.end()) {
// Ставим задачу загрузить с диска.
toDisk.emplace_back(hash, iter->second.front());
} else {
// Сделаем запрос в кеш.
toCache.push_back(hash);
}
}
// Запоминаем, что эти ресурсы уже ожидаются.
WaitingHashes.insert_range(needHashes);
// Запрос в кеш (если там не найдётся, то запрос уйдёт на сервер).
if(!toCache.empty())
Cache->pushReads(std::move(toCache));
// Запрос к диску.
if(!toDisk.empty())
NeedToReadFromDisk.append_range(std::move(toDisk));
_onHashLoad(orr.Files);
}
return result;
}
// ServerSession
// Новые привязки ассетов к Домен+Ключ.
void pushAssetsBindDK(
const std::vector<std::string>& domains,
const std::array<
std::vector<std::vector<std::string>>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
>& keys
) {
LOG.debug() << "BindDK domains=" << domains.size();
for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); ++type) {
LOG.info() << type;
for(size_t forDomainIter = 0; forDomainIter < keys[type].size(); ++forDomainIter) {
LOG.info() << "\t" << domains[forDomainIter];
for(const std::string& key : keys[type][forDomainIter]) {
uint32_t id = getId((EnumAssets) type, domains[forDomainIter], key);
LOG.info() << "\t\t" << key << " -> " << id;
ServerToClientMap[type].push_back(id);
}
}
}
}
// Новые привязки ассетов к Hash+Header.
void pushAssetsBindHH(
std::array<
std::vector<std::tuple<ResourceId, ResourceFile::Hash_t, ResourceHeader>>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
>&& hash_and_headers
) {
std::unordered_set<ResourceFile::Hash_t> needHashes;
size_t totalBinds = 0;
for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); ++type) {
size_t maxSize = 0;
for(auto& [id, hash, header] : hash_and_headers[type]) {
totalBinds++;
assert(id < ServerToClientMap[type].size());
id = ServerToClientMap[type][id];
if(id >= maxSize)
maxSize = id+1;
// Добавляем идентификатор в таблицу ожидающих обновлений.
PendingUpdateFromAsync[type].push_back(id);
// Поискать есть ли ресурс в ресурспаках.
std::optional<AssetsPreloader::Out_Resource> res = ResourcePacks.getResource((EnumAssets) type, id);
if(res) {
needHashes.insert(res->Hash);
} else {
needHashes.insert(hash);
}
}
{
// Уберём повторения в идентификаторах.
auto& vec = PendingUpdateFromAsync[type];
std::sort(vec.begin(), vec.end());
vec.erase(std::unique(vec.begin(), vec.end()), vec.end());
}
if(ServerIdToHH[type].size() < maxSize)
ServerIdToHH[type].resize(maxSize);
for(auto& [id, hash, header] : hash_and_headers[type]) {
ServerIdToHH[type][id] = {hash, std::move(header)};
}
}
if(totalBinds)
LOG.debug() << "BindHH total=" << totalBinds << " wait=" << WaitingHashes.size();
// Нужно убрать хеши, которые уже запрошены
// needHashes ^ WaitingHashes.
for(const auto& hash : WaitingHashes)
needHashes.erase(hash);
std::vector<std::tuple<ResourceFile::Hash_t, fs::path>> toDisk;
std::vector<ResourceFile::Hash_t> toCache;
// Теперь раскидаем хеши по доступным источникам.
for(const auto& hash : needHashes) {
auto iter = HashToPath.find(hash);
if(iter != HashToPath.end()) {
// Ставим задачу загрузить с диска.
toDisk.emplace_back(hash, iter->second.front());
} else {
// Сделаем запрос в кеш.
toCache.push_back(hash);
}
}
// Запоминаем, что эти ресурсы уже ожидаются.
WaitingHashes.insert_range(needHashes);
// Запрос к диску.
if(!toDisk.empty())
NeedToReadFromDisk.append_range(std::move(toDisk));
// Запрос в кеш (если там не найдётся, то запрос уйдёт на сервер).
if(!toCache.empty())
Cache->pushReads(std::move(toCache));
}
// Новые ресурсы, полученные с сервера.
void pushNewResources(
std::vector<std::tuple<ResourceFile::Hash_t, std::u8string>> &&resources
) {
std::unordered_map<ResourceFile::Hash_t, std::u8string> files;
std::vector<Resource> vec;
files.reserve(resources.size());
vec.reserve(resources.size());
for(auto& [hash, res] : resources) {
vec.emplace_back(res);
files.emplace(hash, std::move(res));
}
_onHashLoad(files);
Cache->pushResources(std::move(vec));
}
// Для запроса отсутствующих ресурсов с сервера на клиент.
std::vector<ResourceFile::Hash_t> pullNeededResources() {
return std::move(NeedToRequestFromServer);
}
// Получить изменённые ресурсы (для передачи другим модулям).
ResourceUpdates pullResourceUpdates() {
return std::move(RU);
}
ResourceId reBind(EnumAssets type, ResourceId server) {
return ServerToClientMap[static_cast<size_t>(type)].at(server);
}
void tick() {
// Проверим кеш
std::vector<std::pair<Hash_t, std::optional<Resource>>> resources = Cache->pullReads();
if(!resources.empty()) {
std::unordered_map<ResourceFile::Hash_t, std::u8string> needToProceed;
needToProceed.reserve(resources.size());
for(auto& [hash, res] : resources) {
if(!res)
NeedToRequestFromServer.push_back(hash);
else
needToProceed.emplace(hash, std::u8string{(const char8_t*) res->data(), res->size()});
}
if(!NeedToRequestFromServer.empty())
LOG.debug() << "CacheMiss count=" << NeedToRequestFromServer.size();
if(!needToProceed.empty())
_onHashLoad(needToProceed);
}
/// Читаем с диска TODO: получилась хрень с определением типа, чтобы получать headless ресурс
if(!NeedToReadFromDisk.empty()) {
std::unordered_map<ResourceFile::Hash_t, std::u8string> files;
files.reserve(NeedToReadFromDisk.size());
auto detectTypeDomainKey = [&](const fs::path& path, EnumAssets& typeOut, std::string& domainOut, std::string& keyOut) -> bool {
fs::path cur = path.parent_path();
for(; !cur.empty(); cur = cur.parent_path()) {
std::string name = cur.filename().string();
for(size_t typeIndex = 0; typeIndex < static_cast<size_t>(EnumAssets::MAX_ENUM); ++typeIndex) {
EnumAssets type = static_cast<EnumAssets>(typeIndex);
if(name == ::EnumAssetsToDirectory(type)) {
typeOut = type;
domainOut = cur.parent_path().filename().string();
keyOut = fs::relative(path, cur).generic_string();
return true;
}
}
}
return false;
};
for(const auto& [hash, path] : NeedToReadFromDisk) {
std::u8string data;
std::ifstream file(path, std::ios::binary);
if(file) {
file.seekg(0, std::ios::end);
std::streamoff size = file.tellg();
if(size < 0)
size = 0;
file.seekg(0, std::ios::beg);
data.resize(static_cast<size_t>(size));
if(size > 0) {
file.read(reinterpret_cast<char*>(data.data()), size);
if(!file)
data.clear();
}
} else {
LOG.warn() << "DiskReadFail " << path.string();
}
if(!data.empty()) {
EnumAssets type{};
std::string domain;
std::string key;
if(detectTypeDomainKey(path, type, domain, key)) {
if(type == EnumAssets::Nodestate) {
std::string_view view(reinterpret_cast<const char*>(data.data()), data.size());
js::object obj = js::parse(view).as_object();
HeadlessNodeState hns;
auto modelResolver = [&](const std::string_view model) -> AssetsModel {
auto [mDomain, mKey] = parseDomainKey(model, domain);
return getId(EnumAssets::Model, mDomain, mKey);
};
hns.parse(obj, modelResolver);
data = hns.dump();
} else if(type == EnumAssets::Model) {
std::string_view view(reinterpret_cast<const char*>(data.data()), data.size());
js::object obj = js::parse(view).as_object();
HeadlessModel hm;
auto modelResolver = [&](const std::string_view model) -> AssetsModel {
auto [mDomain, mKey] = parseDomainKey(model, domain);
return getId(EnumAssets::Model, mDomain, mKey);
};
auto textureIdResolver = [&](const std::string_view texture) -> std::optional<uint32_t> {
auto [tDomain, tKey] = parseDomainKey(texture, domain);
return getId(EnumAssets::Texture, tDomain, tKey);
};
auto textureResolver = [&](const std::string_view texturePipelineSrc) -> std::vector<uint8_t> {
TexturePipelineProgram tpp;
if(!tpp.compile(texturePipelineSrc))
return {};
tpp.link(textureIdResolver);
return tpp.toBytes();
};
hm.parse(obj, modelResolver, textureResolver);
data = hm.dump();
}
}
}
files.emplace(hash, std::move(data));
}
NeedToReadFromDisk.clear();
_onHashLoad(files);
}
}
private: private:
struct DomainKey {
std::string Domain;
std::string Key;
bool Known = false;
};
using IdTable = std::unordered_map<
std::string,
std::unordered_map<std::string, AssetId, detail::TSVHash, detail::TSVEq>,
detail::TSVHash,
detail::TSVEq>;
using PackTable = std::unordered_map<
std::string,
std::unordered_map<std::string, PackResource, detail::TSVHash, detail::TSVEq>,
detail::TSVHash,
detail::TSVEq>;
AssetsManager(asio::io_context& ioc, const fs::path& cachePath,
size_t maxCacheDirectorySize, size_t maxLifeTime);
AssetId allocateLocalId(AssetType type);
AssetId ensureServerLocalId(AssetType type, AssetId serverId);
AssetId resolveLocalIdMutable(AssetType type, AssetId localId);
AssetId resolveLocalId(AssetType type, AssetId localId) const;
void unionLocalIds(AssetType type, AssetId fromId, AssetId toId, std::optional<AssetId>* reboundFrom);
std::optional<PackResource> findPackResource(AssetType type, std::string_view domain, std::string_view key) const;
Logger LOG = "Client>AssetsManager"; Logger LOG = "Client>AssetsManager";
// Менеджеры учёта дисковых ресурсов
AssetsPreloader
// В приоритете ищутся ресурсы из ресурспаков по Domain+Key.
ResourcePacks,
/*
Дополнительные источники ресурсов.
Используется для поиска ресурса по хешу от сервера (может стоит тот же мод с совпадающими ресурсами),
или для временной подгрузки ресурса по Domain+Key пока ресурс не был получен с сервера.
*/
ExtraSource;
// Менеджер файлового кэша.
AssetsCacheManager::Ptr Cache; AssetsCacheManager::Ptr Cache;
std::array<IdTable, static_cast<size_t>(AssetType::MAX_ENUM)> DKToLocal; // Указатели на доступные ресурсы
std::array<std::vector<DomainKey>, static_cast<size_t>(AssetType::MAX_ENUM)> LocalToDK; std::unordered_map<ResourceFile::Hash_t, std::vector<fs::path>> HashToPath;
std::array<std::vector<AssetId>, static_cast<size_t>(AssetType::MAX_ENUM)> LocalParent;
std::array<std::vector<AssetId>, static_cast<size_t>(AssetType::MAX_ENUM)> ServerToLocal;
std::array<std::vector<std::optional<BindInfo>>, static_cast<size_t>(AssetType::MAX_ENUM)> BindInfos;
std::array<PackTable, static_cast<size_t>(AssetType::MAX_ENUM)> PackResources;
std::array<AssetId, static_cast<size_t>(AssetType::MAX_ENUM)> NextLocalId{};
std::unordered_map<Hash_t, std::vector<ResourceKey>> PendingReadsByHash; // Таблица релинковки ассетов с идентификаторов сервера на клиентские.
std::vector<std::pair<ResourceKey, std::optional<Resource>>> ReadyReads; std::array<
std::vector<ResourceId>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
> ServerToClientMap;
// Таблица серверных привязок HH (id клиентские)
std::array<
std::vector<std::tuple<ResourceFile::Hash_t, ResourceHeader>>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
> ServerIdToHH;
// Ресурсы в ожидании данных по хешу для обновления (с диска, кеша, сервера).
std::array<
std::vector<ResourceId>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
> PendingUpdateFromAsync;
// Хеши, для которых где-то висит задача на загрузку.
std::unordered_set<ResourceFile::Hash_t> WaitingHashes;
// Хеши, которые необходимо запросить с сервера.
std::vector<ResourceFile::Hash_t> NeedToRequestFromServer;
// Ресурсы, которые нужно считать с диска
std::vector<std::tuple<ResourceFile::Hash_t, fs::path>> NeedToReadFromDisk;
// Обновлённые ресурсы
ResourceUpdates RU;
// Когда данные были получены с диска, кеша или сервера
void _onHashLoad(const std::unordered_map<ResourceFile::Hash_t, std::u8string>& files) {
const auto& rpLinks = ResourcePacks.getResourceLinks();
const auto& esLinks = ExtraSource.getResourceLinks();
auto mapModelId = [&](ResourceId id) -> ResourceId {
const auto& map = ServerToClientMap[static_cast<size_t>(EnumAssets::Model)];
if(id >= map.size())
return 0;
return map[id];
};
auto mapTextureId = [&](ResourceId id) -> ResourceId {
const auto& map = ServerToClientMap[static_cast<size_t>(EnumAssets::Texture)];
if(id >= map.size())
return 0;
return map[id];
};
auto rebindHeader = [&](EnumAssets type, const ResourceHeader& header) -> ResourceHeader {
if(header.empty())
return {};
std::vector<uint8_t> bytes;
bytes.resize(header.size());
std::memcpy(bytes.data(), header.data(), header.size());
std::vector<uint8_t> rebound = AssetsHeaderCodec::rebindHeader(
type,
bytes,
mapModelId,
mapTextureId,
[](const std::string&) {}
);
return ResourceHeader(reinterpret_cast<const char8_t*>(rebound.data()), rebound.size());
};
for(size_t typeIndex = 0; typeIndex < static_cast<size_t>(EnumAssets::MAX_ENUM); ++typeIndex) {
auto& pending = PendingUpdateFromAsync[typeIndex];
if(pending.empty())
continue;
std::vector<ResourceId> stillPending;
stillPending.reserve(pending.size());
size_t updated = 0;
size_t missingSource = 0;
size_t missingData = 0;
for(ResourceId id : pending) {
ResourceFile::Hash_t hash{};
ResourceHeader header;
bool hasSource = false;
bool localHeader = false;
if(id < rpLinks[typeIndex].size() && rpLinks[typeIndex][id].IsExist) {
hash = rpLinks[typeIndex][id].Hash;
header = rpLinks[typeIndex][id].Header;
hasSource = true;
localHeader = true;
} else if(id < ServerIdToHH[typeIndex].size()) {
std::tie(hash, header) = ServerIdToHH[typeIndex][id];
hasSource = true;
}
if(!hasSource) {
missingSource++;
stillPending.push_back(id);
continue;
}
auto dataIter = files.find(hash);
if(dataIter == files.end()) {
missingData++;
stillPending.push_back(id);
continue;
}
std::string domain = "core";
std::string key;
{
auto d = getDK((EnumAssets) typeIndex, id);
if(d) {
domain = d->Domain;
key = d->Key;
}
}
std::u8string data = dataIter->second;
EnumAssets type = static_cast<EnumAssets>(typeIndex);
ResourceHeader finalHeader = localHeader ? header : rebindHeader(type, header);
if(id == 0)
continue;
if(type == EnumAssets::Nodestate) {
HeadlessNodeState ns;
ns.load(data);
HeadlessNodeState::Header headerParsed;
headerParsed.load(finalHeader);
RU.Nodestates.push_back({id, std::move(ns), std::move(headerParsed)});
updated++;
} else if(type == EnumAssets::Model) {
HeadlessModel hm;
hm.load(data);
HeadlessModel::Header headerParsed;
headerParsed.load(finalHeader);
RU.Models.push_back({id, std::move(hm), std::move(headerParsed)});
updated++;
} else if(type == EnumAssets::Texture) {
AssetsTextureUpdate entry;
entry.Id = id;
entry.Header = std::move(finalHeader);
if(!data.empty()) {
iResource sres(reinterpret_cast<const uint8_t*>(data.data()), data.size());
iBinaryStream stream = sres.makeStream();
png::image<png::rgba_pixel> img(stream.Stream);
entry.Width = static_cast<uint16_t>(img.get_width());
entry.Height = static_cast<uint16_t>(img.get_height());
entry.Pixels.resize(static_cast<size_t>(entry.Width) * entry.Height);
for(uint32_t y = 0; y < entry.Height; ++y) {
const auto& row = img.get_pixbuf().operator[](y);
for(uint32_t x = 0; x < entry.Width; ++x) {
const auto& px = row[x];
uint32_t rgba = (uint32_t(px.alpha) << 24)
| (uint32_t(px.red) << 16)
| (uint32_t(px.green) << 8)
| uint32_t(px.blue);
entry.Pixels[x + y * entry.Width] = rgba;
}
}
}
RU.Textures.push_back(std::move(entry));
updated++;
} else if(type == EnumAssets::Particle) {
RU.Particles.push_back({id, std::move(data)});
updated++;
} else if(type == EnumAssets::Animation) {
RU.Animations.push_back({id, std::move(data)});
updated++;
} else if(type == EnumAssets::Sound) {
RU.Sounds.push_back({id, std::move(data)});
updated++;
} else if(type == EnumAssets::Font) {
RU.Fonts.push_back({id, std::move(data)});
updated++;
}
}
if(updated || missingSource || missingData) {
LOG.debug() << "HashLoad type=" << int(typeIndex)
<< " updated=" << updated
<< " missingSource=" << missingSource
<< " missingData=" << missingData;
}
pending = std::move(stillPending);
}
for(const auto& [hash, res] : files)
WaitingHashes.erase(hash);
}
}; };
} // namespace LV::Client } // namespace LV::Client

File diff suppressed because it is too large Load Diff

View File

@@ -69,22 +69,26 @@ private:
IRenderSession *RS = nullptr; IRenderSession *RS = nullptr;
// Обработчик кеша ресурсов сервера // Обработчик кеша ресурсов сервера
AssetsManager::Ptr AM; AssetsManager AM;
static constexpr uint64_t TIME_BEFORE_UNLOAD_RESOURCE = 180; static constexpr uint64_t TIME_BEFORE_UNLOAD_RESOURCE = 180;
struct { struct {
// Существующие привязки ресурсов // Существующие привязки ресурсов
std::unordered_set<ResourceId> ExistBinds[(int) EnumAssets::MAX_ENUM]; // std::unordered_set<ResourceId> ExistBinds[(int) EnumAssets::MAX_ENUM];
// Недавно использованные ресурсы, пока хранятся здесь в течении TIME_BEFORE_UNLOAD_RESOURCE секунд // Недавно использованные ресурсы, пока хранятся здесь в течении TIME_BEFORE_UNLOAD_RESOURCE секунд
std::unordered_map<std::string, std::pair<AssetEntry, uint64_t>> NotInUse[(int) EnumAssets::MAX_ENUM]; // std::unordered_map<std::string, std::pair<AssetEntry, uint64_t>> NotInUse[(int) EnumAssets::MAX_ENUM];
} MyAssets; } MyAssets;
struct AssetLoading { struct AssetLoadingEntry {
EnumAssets Type; EnumAssets Type;
ResourceId Id; ResourceId Id;
std::string Domain, Key; std::string Domain;
std::string Key;
};
struct AssetLoading {
std::u8string Data; std::u8string Data;
size_t Offset; size_t Offset = 0;
}; };
struct AssetBindEntry { struct AssetBindEntry {
@@ -95,21 +99,41 @@ private:
std::vector<uint8_t> Header; std::vector<uint8_t> Header;
}; };
std::array<std::vector<std::pair<std::string, std::string>>, (int) EnumAssets::MAX_ENUM> ServerIdToDK; struct UpdateAssetsBindsDK {
std::array<ResourceId, (int) EnumAssets::MAX_ENUM> NextServerId = {}; std::vector<std::string> Domains;
std::array<
std::vector<std::vector<std::string>>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
> Keys;
};
struct UpdateAssetsBindsHH {
std::array<
std::vector<std::tuple<ResourceId, ResourceFile::Hash_t, ResourceHeader>>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
> HashAndHeaders;
};
struct TickData { struct TickData {
std::vector<std::pair<DefVoxelId, void*>> Profile_Voxel_AddOrChange; // Полученные изменения привязок Domain+Key
std::vector<UpdateAssetsBindsDK> BindsDK;
// Полученные изменения привязок Hash+Header
std::vector<UpdateAssetsBindsHH> BindsHH;
// Потерянные привязываются к hash_t(0)
// Полученные с сервера ресурсы
std::vector<std::tuple<ResourceFile::Hash_t, std::u8string>> ReceivedAssets;
std::vector<std::pair<DefVoxelId, DefVoxel>> Profile_Voxel_AddOrChange;
std::vector<DefVoxelId> Profile_Voxel_Lost; std::vector<DefVoxelId> Profile_Voxel_Lost;
std::vector<std::pair<DefNodeId, DefNode_t>> Profile_Node_AddOrChange; std::vector<std::pair<DefNodeId, DefNode>> Profile_Node_AddOrChange;
std::vector<DefNodeId> Profile_Node_Lost; std::vector<DefNodeId> Profile_Node_Lost;
std::vector<std::pair<DefWorldId, void*>> Profile_World_AddOrChange; std::vector<std::pair<DefWorldId, DefWorld>> Profile_World_AddOrChange;
std::vector<DefWorldId> Profile_World_Lost; std::vector<DefWorldId> Profile_World_Lost;
std::vector<std::pair<DefPortalId, void*>> Profile_Portal_AddOrChange; std::vector<std::pair<DefPortalId, DefPortal>> Profile_Portal_AddOrChange;
std::vector<DefPortalId> Profile_Portal_Lost; std::vector<DefPortalId> Profile_Portal_Lost;
std::vector<std::pair<DefEntityId, DefEntityInfo>> Profile_Entity_AddOrChange; std::vector<std::pair<DefEntityId, DefEntity>> Profile_Entity_AddOrChange;
std::vector<DefEntityId> Profile_Entity_Lost; std::vector<DefEntityId> Profile_Entity_Lost;
std::vector<std::pair<DefItemId, void*>> Profile_Item_AddOrChange; std::vector<std::pair<DefItemId, DefItem>> Profile_Item_AddOrChange;
std::vector<DefItemId> Profile_Item_Lost; std::vector<DefItemId> Profile_Item_Lost;
std::vector<std::pair<WorldId_t, void*>> Worlds_AddOrChange; std::vector<std::pair<WorldId_t, void*>> Worlds_AddOrChange;
@@ -127,13 +151,6 @@ private:
uint32_t Nodes = 0; uint32_t Nodes = 0;
}; };
struct AssetsBindsChange {
// Новые привязки ресурсов
std::vector<AssetBindEntry> Binds;
// Потерянные из видимости ресурсы
std::vector<ResourceId> Lost[(int) EnumAssets::MAX_ENUM];
};
struct { struct {
// Сюда обращается ветка, обрабатывающая сокет; run() // Сюда обращается ветка, обрабатывающая сокет; run()
// Получение ресурсов с сервера // Получение ресурсов с сервера
@@ -141,22 +158,7 @@ private:
// Накопление данных за такт сервера // Накопление данных за такт сервера
TickData ThisTickEntry; TickData ThisTickEntry;
// Сюда обращается ветка обновления IServerSession, накапливая данные до SyncTick
// Ресурсы, ожидающие ответа от менеджера кеша
std::unordered_map<std::string, std::vector<std::pair<std::string, Hash_t>>> ResourceWait[(int) EnumAssets::MAX_ENUM];
// Полученные изменения связок в ожидании стадии синхронизации такта
std::vector<AssetsBindsChange> Binds;
// Подгруженные или принятые меж тактами ресурсы
std::vector<AssetEntry> LoadedResources;
// Список ресурсов на которые уже был отправлен запрос на загрузку ресурса
std::vector<Hash_t> AlreadyLoading;
// Обменный пункт // Обменный пункт
// Полученные ресурсы с сервера
TOS::SpinlockObject<std::vector<AssetEntry>> LoadedAssets;
// Изменения в наблюдаемых ресурсах
TOS::SpinlockObject<std::vector<AssetsBindsChange>> AssetsBinds;
// Пакеты обновлений игрового мира // Пакеты обновлений игрового мира
TOS::SpinlockObject<std::vector<TickData>> TickSequence; TOS::SpinlockObject<std::vector<TickData>> TickSequence;
} AsyncContext; } AsyncContext;
@@ -196,6 +198,7 @@ private:
coro<> rP_AssetsBindHH(Net::AsyncSocket &sock); coro<> rP_AssetsBindHH(Net::AsyncSocket &sock);
coro<> rP_AssetsInitSend(Net::AsyncSocket &sock); coro<> rP_AssetsInitSend(Net::AsyncSocket &sock);
coro<> rP_AssetsNextSend(Net::AsyncSocket &sock); coro<> rP_AssetsNextSend(Net::AsyncSocket &sock);
coro<> rP_DefinitionsFull(Net::AsyncSocket &sock);
coro<> rP_DefinitionsUpdate(Net::AsyncSocket &sock); coro<> rP_DefinitionsUpdate(Net::AsyncSocket &sock);
coro<> rP_ChunkVoxels(Net::AsyncSocket &sock); coro<> rP_ChunkVoxels(Net::AsyncSocket &sock);
coro<> rP_ChunkNodes(Net::AsyncSocket &sock); coro<> rP_ChunkNodes(Net::AsyncSocket &sock);

View File

@@ -16,7 +16,7 @@
class SharedStagingBuffer { class SharedStagingBuffer {
public: public:
static constexpr VkDeviceSize kDefaultSize = 64ull * 1024ull * 1024ull; static constexpr VkDeviceSize kDefaultSize = 18ull * 1024ull * 1024ull;
SharedStagingBuffer(VkDevice device, SharedStagingBuffer(VkDevice device,
VkPhysicalDevice physicalDevice, VkPhysicalDevice physicalDevice,

View File

@@ -29,6 +29,7 @@
#include <png++/png.hpp> #include <png++/png.hpp>
#include "VulkanRenderSession.hpp" #include "VulkanRenderSession.hpp"
#include <Server/GameServer.hpp> #include <Server/GameServer.hpp>
#include <malloc.h>
extern void LoadSymbolsVulkan(TOS::DynamicLibrary &library); extern void LoadSymbolsVulkan(TOS::DynamicLibrary &library);
@@ -222,6 +223,8 @@ void Vulkan::run()
} catch(const std::exception &exc) { } catch(const std::exception &exc) {
LOG.error() << "Game.Session->shutdown: " << exc.what(); LOG.error() << "Game.Session->shutdown: " << exc.what();
} }
Game.Session = nullptr;
} }
if(!NeedShutdown && glfwWindowShouldClose(Graphics.Window)) { if(!NeedShutdown && glfwWindowShouldClose(Graphics.Window)) {
@@ -240,11 +243,12 @@ void Vulkan::run()
try { try {
if(Game.Session) if(Game.Session)
Game.Session->shutdown(EnumDisconnect::ByInterface); Game.Session->shutdown(EnumDisconnect::ByInterface);
Game.Session = nullptr;
} catch(const std::exception &exc) { } catch(const std::exception &exc) {
LOG.error() << "Game.Session->shutdown: " << exc.what(); LOG.error() << "Game.Session->shutdown: " << exc.what();
} }
Game.Session = nullptr;
try { try {
if(Game.Server) if(Game.Server)
Game.Server->GS.shutdown("Завершение работы из-за остановки клиента"); Game.Server->GS.shutdown("Завершение работы из-за остановки клиента");
@@ -2254,6 +2258,10 @@ void Vulkan::gui_MainMenu() {
} }
} }
if(ImGui::Button("Memory trim")) {
malloc_trim(0);
}
if(ConnectionProgress.InProgress) { if(ConnectionProgress.InProgress) {
if(ImGui::Button("Отмена")) if(ImGui::Button("Отмена"))
ConnectionProgress.Cancel = true; ConnectionProgress.Cancel = true;
@@ -2291,6 +2299,8 @@ void Vulkan::gui_ConnectedToServer() {
double chunksKb = double(Game.Session->getVisibleCompressedChunksBytes()) / 1024.0; double chunksKb = double(Game.Session->getVisibleCompressedChunksBytes()) / 1024.0;
ImGui::Text("chunks compressed: %.1f KB", chunksKb); ImGui::Text("chunks compressed: %.1f KB", chunksKb);
ImGui::Checkbox("Логи сетевых пакетов", &Game.Session->DebugLogPackets);
if(ImGui::Button("Delimeter")) if(ImGui::Button("Delimeter"))
LOG.debug(); LOG.debug();
@@ -2303,6 +2313,10 @@ void Vulkan::gui_ConnectedToServer() {
Game.ImGuiInterfaces.pop_back(); Game.ImGuiInterfaces.pop_back();
} }
if(ImGui::Button("Memory trim")) {
malloc_trim(0);
}
ImGui::End(); ImGui::End();
if(Game.Выйти) if(Game.Выйти)

View File

@@ -48,7 +48,7 @@ struct DeviceId {
struct Settings { struct Settings {
DeviceId DeviceMain; DeviceId DeviceMain;
uint32_t QueueGraphics = -1, QueueSurface = -1; uint32_t QueueGraphics = -1, QueueSurface = -1;
bool Debug = true; bool Debug = false;
bool isValid() bool isValid()
{ {

View File

@@ -121,15 +121,15 @@ void ChunkMeshGenerator::run(uint8_t id) {
uint8_t fullNodes[18][18][18]; uint8_t fullNodes[18][18][18];
// Профиль, который используется если на стороне клиента отсутствует нужных профиль // Профиль, который используется если на стороне клиента отсутствует нужных профиль
DefNode_t defaultProfileNode; DefNode defaultProfileNode;
// Кеш запросов профилей нод // Кеш запросов профилей нод
std::unordered_map<DefNodeId, const DefNode_t*> profilesNodeCache; std::unordered_map<DefNodeId, const DefNode*> profilesNodeCache;
auto getNodeProfile = [&](DefNodeId id) -> const DefNode_t* { auto getNodeProfile = [&](DefNodeId id) -> const DefNode* {
auto iterCache = profilesNodeCache.find(id); auto iterCache = profilesNodeCache.find(id);
if(iterCache == profilesNodeCache.end()) { if(iterCache == profilesNodeCache.end()) {
// Промах кеша // Промах кеша
auto iterSS = SS->Profiles.DefNode.find(id); auto iterSS = SS->Profiles.DefNodes.find(id);
if(iterSS != SS->Profiles.DefNode.end()) { if(iterSS != SS->Profiles.DefNodes.end()) {
return (profilesNodeCache[id] = &iterSS->second); return (profilesNodeCache[id] = &iterSS->second);
} else { } else {
// Профиль отсутствует на клиенте // Профиль отсутствует на клиенте
@@ -140,6 +140,14 @@ void ChunkMeshGenerator::run(uint8_t id) {
} }
}; };
std::vector<NodeStateInfo> metaStatesInfo;
{
NodeStateInfo info;
info.Name = "meta";
info.Variations = 256;
metaStatesInfo.push_back(std::move(info));
}
// Воксели пока не рендерим // Воксели пока не рендерим
if(auto iterWorld = SS->Content.Worlds.find(wId); iterWorld != SS->Content.Worlds.end()) { if(auto iterWorld = SS->Content.Worlds.find(wId); iterWorld != SS->Content.Worlds.end()) {
Pos::GlobalRegion rPos = pos >> 2; Pos::GlobalRegion rPos = pos >> 2;
@@ -181,14 +189,50 @@ void ChunkMeshGenerator::run(uint8_t id) {
std::unordered_map<DefNodeId, bool> nodeFullCuboidCache; std::unordered_map<DefNodeId, bool> nodeFullCuboidCache;
auto nodeIsFull = [&](Node node) -> bool { auto nodeIsFull = [&](Node node) -> bool {
if(node.NodeId == 0)
return false;
auto iterCache = nodeFullCuboidCache.find(node.Data); auto iterCache = nodeFullCuboidCache.find(node.Data);
if(iterCache == nodeFullCuboidCache.end()) { if(iterCache == nodeFullCuboidCache.end()) {
const DefNode_t* profile = getNodeProfile(node.NodeId); const DefNode* profile = getNodeProfile(node.NodeId);
if(profile->TexId != 0) { if(NSP) {
return (nodeFullCuboidCache[node.Data] = true); if(const AssetsNodestate* ptr = std::get_if<AssetsNodestate>(&profile->RenderStates)) {
if(NSP->hasNodestate(*ptr)) {
std::unordered_map<std::string, int32_t> states;
int32_t meta = node.Meta;
states.emplace("meta", meta);
const auto routes = NSP->getModelsForNode(*ptr, metaStatesInfo, states);
bool isFull = !routes.empty();
if(isFull) {
for(const auto& variants : routes) {
for(const auto& [weight, faces] : variants) {
(void)weight;
auto hasFace = [&](EnumFace face) -> bool {
auto iterFace = faces.find(face);
return iterFace != faces.end() && !iterFace->second.empty();
};
if(!hasFace(EnumFace::Up)
|| !hasFace(EnumFace::Down)
|| !hasFace(EnumFace::East)
|| !hasFace(EnumFace::West)
|| !hasFace(EnumFace::South)
|| !hasFace(EnumFace::North))
{
isFull = false;
break;
}
}
if(!isFull)
break;
}
}
return (nodeFullCuboidCache[node.Data] = isFull);
}
}
} }
return (nodeFullCuboidCache[node.Data] = false); return (nodeFullCuboidCache[node.Data] = true);
} else { } else {
return iterCache->second; return iterCache->second;
} }
@@ -273,7 +317,7 @@ void ChunkMeshGenerator::run(uint8_t id) {
} else { } else {
for(int y = 0; y < 16; y++) for(int y = 0; y < 16; y++)
for(int x = 0; x < 16; x++) for(int x = 0; x < 16; x++)
fullNodes[x+0][y+1][0] = 0; fullNodes[x+1][y+1][0] = 0;
} }
} else } else
goto end; goto end;
@@ -325,14 +369,6 @@ void ChunkMeshGenerator::run(uint8_t id) {
std::unordered_map<uint32_t, ModelCacheEntry> modelCache; std::unordered_map<uint32_t, ModelCacheEntry> modelCache;
std::unordered_map<AssetsTexture, uint32_t> baseTextureCache; std::unordered_map<AssetsTexture, uint32_t> baseTextureCache;
std::vector<NodeStateInfo> metaStatesInfo;
{
NodeStateInfo info;
info.Name = "meta";
info.Variations = 256;
metaStatesInfo.push_back(std::move(info));
}
auto isFaceCovered = [&](EnumFace face, int covered) -> bool { auto isFaceCovered = [&](EnumFace face, int covered) -> bool {
switch(face) { switch(face) {
case EnumFace::Up: return covered & (1 << 2); case EnumFace::Up: return covered & (1 << 2);
@@ -388,6 +424,10 @@ void ChunkMeshGenerator::run(uint8_t id) {
for(int z = 0; z < 16; z++) for(int z = 0; z < 16; z++)
for(int y = 0; y < 16; y++) for(int y = 0; y < 16; y++)
for(int x = 0; x < 16; x++) { for(int x = 0; x < 16; x++) {
const Node& nodeData = (*chunk)[x+y*16+z*16*16];
if(nodeData.NodeId == 0)
continue;
const size_t vertexStart = result.NodeVertexs.size(); const size_t vertexStart = result.NodeVertexs.size();
int fullCovered = 0; int fullCovered = 0;
@@ -401,66 +441,46 @@ void ChunkMeshGenerator::run(uint8_t id) {
if(fullCovered == 0b111111) if(fullCovered == 0b111111)
continue; continue;
const Node& nodeData = (*chunk)[x+y*16+z*16*16]; const DefNode* node = getNodeProfile(nodeData.NodeId);
const DefNode_t* node = getNodeProfile(nodeData.NodeId);
if(debugMeshEnabled) {
const bool hasRenderable = (node->NodestateId != 0) || (node->TexId != 0);
if(hasRenderable && fullCovered != 0b111111) {
expectedColumnX[x] = 1;
expectedColumnY[y] = 1;
expectedColumnZ[z] = 1;
}
}
bool usedModel = false; bool usedModel = false;
if(NSP && (node->NodestateId != 0 || NSP->hasNodestate(node->NodestateId))) { if(NSP) {
auto iterCache = modelCache.find(nodeData.Data); if(const AssetsNodestate* ptr = std::get_if<AssetsNodestate>(&node->RenderStates)) {
if(iterCache == modelCache.end()) { if(NSP->hasNodestate(*ptr)) {
std::unordered_map<std::string, int32_t> states; auto iterCache = modelCache.find(nodeData.Data);
states.emplace("meta", nodeData.Meta); if(iterCache == modelCache.end()) {
std::unordered_map<std::string, int32_t> states;
states.emplace("meta", nodeData.Meta);
ModelCacheEntry entry; ModelCacheEntry entry;
entry.Routes = NSP->getModelsForNode(node->NodestateId, metaStatesInfo, states); entry.Routes = NSP->getModelsForNode(*ptr, metaStatesInfo, states);
iterCache = modelCache.emplace(nodeData.Data, std::move(entry)).first; iterCache = modelCache.emplace(nodeData.Data, std::move(entry)).first;
} }
if(!iterCache->second.Routes.empty()) { if(!iterCache->second.Routes.empty()) {
uint32_t seed = uint32_t(nodeData.Data) * 2654435761u; uint32_t seed = uint32_t(nodeData.Data) * 2654435761u;
seed ^= uint32_t(x) * 73856093u; seed ^= uint32_t(x) * 73856093u;
seed ^= uint32_t(y) * 19349663u; seed ^= uint32_t(y) * 19349663u;
seed ^= uint32_t(z) * 83492791u; seed ^= uint32_t(z) * 83492791u;
for(size_t routeIndex = 0; routeIndex < iterCache->second.Routes.size(); routeIndex++) { for(size_t routeIndex = 0; routeIndex < iterCache->second.Routes.size(); routeIndex++) {
const auto& variants = iterCache->second.Routes[routeIndex]; const auto& variants = iterCache->second.Routes[routeIndex];
const auto* faces = pickVariant(variants, seed + uint32_t(routeIndex) * 374761393u); const auto* faces = pickVariant(variants, seed + uint32_t(routeIndex) * 374761393u);
if(faces) if(faces)
appendModel(*faces, fullCovered, x, y, z); appendModel(*faces, fullCovered, x, y, z);
}
usedModel = true;
}
} }
usedModel = true;
} }
} }
if(usedModel) if(usedModel)
goto node_done; goto node_done;
if(NSP && node->TexId != 0) { v.Tex = 0;
auto iterTex = baseTextureCache.find(node->TexId);
if(iterTex != baseTextureCache.end()) {
v.Tex = iterTex->second;
} else {
uint32_t resolvedTex = NSP->getTextureId(node->TexId);
v.Tex = resolvedTex;
baseTextureCache.emplace(node->TexId, resolvedTex);
}
} else {
v.Tex = node->TexId;
}
if(v.Tex == 0)
goto node_done;
// Рендерим обычный кубоид // Рендерим обычный кубоид
// XZ+Y // XZ+Y
@@ -668,58 +688,6 @@ void ChunkMeshGenerator::run(uint8_t id) {
} }
node_done: node_done:
if(debugMeshEnabled) {
const bool emitted = result.NodeVertexs.size() > vertexStart;
if(emitted) {
generatedColumnX[x] = 1;
generatedColumnY[y] = 1;
generatedColumnZ[z] = 1;
} else {
const bool hasRenderable = (node->NodestateId != 0) || (node->TexId != 0);
if(hasRenderable && fullCovered != 0b111111) {
uint32_t warnIndex = debugMeshWarnCount.fetch_add(1);
if(warnIndex < 16) {
LOG.warn() << "Missing node geometry at chunk " << int(pos[0]) << ','
<< int(pos[1]) << ',' << int(pos[2])
<< " local " << x << ',' << y << ',' << z
<< " nodeId " << nodeData.NodeId
<< " meta " << int(nodeData.Meta)
<< " covered " << fullCovered
<< " tex " << node->TexId
<< " nodestate " << node->NodestateId;
}
}
}
}
}
if(debugMeshEnabled) {
auto collectMissing = [](const std::array<uint8_t, 16>& expected,
const std::array<uint8_t, 16>& generated) {
std::string res;
for(int i = 0; i < 16; i++) {
if(expected[i] && !generated[i]) {
if(!res.empty())
res += ',';
res += std::to_string(i);
}
}
return res;
};
std::string missingX = collectMissing(expectedColumnX, generatedColumnX);
std::string missingY = collectMissing(expectedColumnY, generatedColumnY);
std::string missingZ = collectMissing(expectedColumnZ, generatedColumnZ);
if(!missingX.empty() || !missingY.empty() || !missingZ.empty()) {
uint32_t warnIndex = debugMeshWarnCount.fetch_add(1);
if(warnIndex < 16) {
LOG.warn() << "Missing mesh columns at chunk " << int(pos[0]) << ','
<< int(pos[1]) << ',' << int(pos[2])
<< " missingX[" << missingX << "]"
<< " missingY[" << missingY << "]"
<< " missingZ[" << missingZ << "]";
}
}
} }
// Вычислить индексы и сократить вершины // Вычислить индексы и сократить вершины
@@ -907,6 +875,7 @@ void ChunkPreparator::tickSync(const TickSyncData& data) {
// Получаем готовые чанки // Получаем готовые чанки
{ {
std::vector<ChunkMeshGenerator::ChunkObj_t> chunks = std::move(*CMG.Output.lock()); std::vector<ChunkMeshGenerator::ChunkObj_t> chunks = std::move(*CMG.Output.lock());
uint8_t frameRetirement = (FrameRoulette+FRAME_COUNT_RESOURCE_LATENCY) % FRAME_COUNT_RESOURCE_LATENCY;
for(auto& chunk : chunks) { for(auto& chunk : chunks) {
auto iterWorld = Requests.find(chunk.WId); auto iterWorld = Requests.find(chunk.WId);
if(iterWorld == Requests.end()) if(iterWorld == Requests.end())
@@ -921,6 +890,14 @@ void ChunkPreparator::tickSync(const TickSyncData& data) {
// Чанк ожидаем // Чанк ожидаем
auto& rChunk = ChunksMesh[chunk.WId][chunk.Pos >> 2][Pos::bvec4u(chunk.Pos & 0x3).pack()]; auto& rChunk = ChunksMesh[chunk.WId][chunk.Pos >> 2][Pos::bvec4u(chunk.Pos & 0x3).pack()];
if(rChunk.VoxelPointer)
VPV_ToFree[frameRetirement].emplace_back(std::move(rChunk.VoxelPointer));
if(rChunk.NodePointer) {
VPN_ToFree[frameRetirement].emplace_back(std::move(rChunk.NodePointer), std::move(rChunk.NodeIndexes));
}
rChunk.VoxelPointer = {};
rChunk.NodePointer = {};
rChunk.NodeIndexes = {};
rChunk.Voxels = std::move(chunk.VoxelDefines); rChunk.Voxels = std::move(chunk.VoxelDefines);
if(!chunk.VoxelVertexs.empty()) if(!chunk.VoxelVertexs.empty())
rChunk.VoxelPointer = VertexPool_Voxels.pushVertexs(std::move(chunk.VoxelVertexs)); rChunk.VoxelPointer = VertexPool_Voxels.pushVertexs(std::move(chunk.VoxelVertexs));
@@ -1603,6 +1580,8 @@ VulkanRenderSession::VulkanRenderSession(Vulkan *vkInst, IServerSession *serverS
} }
VulkanRenderSession::~VulkanRenderSession() { VulkanRenderSession::~VulkanRenderSession() {
if(VoxelOpaquePipeline) if(VoxelOpaquePipeline)
vkDestroyPipeline(VkInst->Graphics.Device, VoxelOpaquePipeline, nullptr); vkDestroyPipeline(VkInst->Graphics.Device, VoxelOpaquePipeline, nullptr);
if(VoxelTransparentPipeline) if(VoxelTransparentPipeline)
@@ -1630,7 +1609,7 @@ void VulkanRenderSession::pushStageTickSync() {
CP.pushStageTickSync(); CP.pushStageTickSync();
} }
void VulkanRenderSession::tickSync(const TickSyncData& data) { void VulkanRenderSession::tickSync(TickSyncData& data) {
// Изменение ассетов // Изменение ассетов
// Профили // Профили
// Чанки // Чанки
@@ -1648,75 +1627,16 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) {
if(auto iter = data.Profiles_Lost.find(EnumDefContent::Voxel); iter != data.Profiles_Lost.end()) if(auto iter = data.Profiles_Lost.find(EnumDefContent::Voxel); iter != data.Profiles_Lost.end())
mcpData.ChangedVoxels.insert(mcpData.ChangedVoxels.end(), iter->second.begin(), iter->second.end()); mcpData.ChangedVoxels.insert(mcpData.ChangedVoxels.end(), iter->second.begin(), iter->second.end());
std::vector<std::tuple<AssetsModel, Resource, const std::vector<uint8_t>*>> modelResources;
std::vector<AssetsModel> modelLost;
if(auto iter = data.Assets_ChangeOrAdd.find(EnumAssets::Model); iter != data.Assets_ChangeOrAdd.end()) {
const auto& list = ServerSession->Assets[EnumAssets::Model];
for(ResourceId id : iter->second) {
auto entryIter = list.find(id);
if(entryIter == list.end())
continue;
modelResources.emplace_back(id, entryIter->second.Res, &entryIter->second.Dependencies);
}
}
if(auto iter = data.Assets_Lost.find(EnumAssets::Model); iter != data.Assets_Lost.end())
modelLost.insert(modelLost.end(), iter->second.begin(), iter->second.end());
std::vector<AssetsModel> changedModels; std::vector<AssetsModel> changedModels;
if(!modelResources.empty() || !modelLost.empty()) { if(!data.AssetsModels.empty())
const auto& modelAssets = ServerSession->Assets[EnumAssets::Model]; changedModels = MP.onModelChanges(std::move(data.AssetsModels));
changedModels = MP.onModelChanges(std::move(modelResources), std::move(modelLost), &modelAssets);
}
if(TP) { if(TP && !data.AssetsTextures.empty())
std::vector<TextureProvider::TextureUpdate> textureResources; TP->onTexturesChanges(std::move(data.AssetsTextures));
std::vector<AssetsTexture> textureLost;
if(auto iter = data.Assets_ChangeOrAdd.find(EnumAssets::Texture); iter != data.Assets_ChangeOrAdd.end()) {
const auto& list = ServerSession->Assets[EnumAssets::Texture];
for(ResourceId id : iter->second) {
auto entryIter = list.find(id);
if(entryIter == list.end())
continue;
textureResources.push_back({
.Id = id,
.Res = entryIter->second.Res,
.Domain = entryIter->second.Domain,
.Key = entryIter->second.Key
});
}
}
if(auto iter = data.Assets_Lost.find(EnumAssets::Texture); iter != data.Assets_Lost.end())
textureLost.insert(textureLost.end(), iter->second.begin(), iter->second.end());
if(!textureResources.empty() || !textureLost.empty())
TP->onTexturesChanges(std::move(textureResources), std::move(textureLost));
}
std::vector<AssetsNodestate> changedNodestates; std::vector<AssetsNodestate> changedNodestates;
if(NSP) { if(NSP && (!data.AssetsNodestates.empty() || !changedModels.empty())) {
std::vector<std::tuple<AssetsNodestate, Resource, const std::vector<uint8_t>*>> nodestateResources; changedNodestates = NSP->onNodestateChanges(std::move(data.AssetsNodestates), std::move(changedModels));
std::vector<AssetsNodestate> nodestateLost;
if(auto iter = data.Assets_ChangeOrAdd.find(EnumAssets::Nodestate); iter != data.Assets_ChangeOrAdd.end()) {
const auto& list = ServerSession->Assets[EnumAssets::Nodestate];
for(ResourceId id : iter->second) {
auto entryIter = list.find(id);
if(entryIter == list.end())
continue;
nodestateResources.emplace_back(id, entryIter->second.Res, &entryIter->second.Dependencies);
}
}
if(auto iter = data.Assets_Lost.find(EnumAssets::Nodestate); iter != data.Assets_Lost.end())
nodestateLost.insert(nodestateLost.end(), iter->second.begin(), iter->second.end());
if(!nodestateResources.empty() || !nodestateLost.empty() || !changedModels.empty())
changedNodestates = NSP->onNodestateChanges(std::move(nodestateResources), std::move(nodestateLost), changedModels);
} }
if(!changedNodestates.empty()) { if(!changedNodestates.empty()) {
@@ -1725,9 +1645,10 @@ void VulkanRenderSession::tickSync(const TickSyncData& data) {
for(AssetsNodestate id : changedNodestates) for(AssetsNodestate id : changedNodestates)
changed.insert(id); changed.insert(id);
for(const auto& [nodeId, def] : ServerSession->Profiles.DefNode) { for(const auto& [nodeId, def] : ServerSession->Profiles.DefNodes) {
if(changed.contains(def.NodestateId)) if(const AssetsNodestate* ptr = std::get_if<AssetsNodestate>(&def.RenderStates))
mcpData.ChangedNodes.push_back(nodeId); if(changed.contains(*ptr))
mcpData.ChangedNodes.push_back(nodeId);
} }
} }
@@ -2016,29 +1937,7 @@ void VulkanRenderSession::ensureEntityTexture() {
if(EntityTextureReady || !TP || !NSP) if(EntityTextureReady || !TP || !NSP)
return; return;
auto iter = ServerSession->Assets.find(EnumAssets::Texture); return;
if(iter == ServerSession->Assets.end() || iter->second.empty())
return;
const AssetEntry* picked = nullptr;
for(const auto& [id, entry] : iter->second) {
if(entry.Key == "default.png") {
picked = &entry;
break;
}
}
if(!picked) {
for(const auto& [id, entry] : iter->second) {
if(entry.Key == "grass.png") {
picked = &entry;
break;
}
}
}
if(!picked)
picked = &iter->second.begin()->second;
updateTestQuadTexture(NSP->getTextureId(picked->Id));
} }
void VulkanRenderSession::ensureAtlasLayerPreview() { void VulkanRenderSession::ensureAtlasLayerPreview() {

View File

@@ -5,6 +5,8 @@
#include "Common/Abstract.hpp" #include "Common/Abstract.hpp"
#include <Client/Vulkan/Vulkan.hpp> #include <Client/Vulkan/Vulkan.hpp>
#include <algorithm> #include <algorithm>
#include <array>
#include <atomic>
#include <bitset> #include <bitset>
#include <condition_variable> #include <condition_variable>
#include <cstring> #include <cstring>
@@ -87,9 +89,7 @@ public:
} }
// Применяет изменения, возвращая все затронутые модели // Применяет изменения, возвращая все затронутые модели
std::vector<AssetsModel> onModelChanges(std::vector<std::tuple<AssetsModel, Resource, const std::vector<uint8_t>*>> newOrChanged, std::vector<AssetsModel> onModelChanges(std::vector<AssetsModelUpdate> entries) {
std::vector<AssetsModel> lost,
const std::unordered_map<ResourceId, AssetEntry>* modelAssets) {
std::vector<AssetsModel> result; std::vector<AssetsModel> result;
std::move_only_function<void(ResourceId)> makeUnready; std::move_only_function<void(ResourceId)> makeUnready;
@@ -121,88 +121,51 @@ public:
iterModel->second.Ready = false; iterModel->second.Ready = false;
}; };
for(ResourceId lostId : lost) {
makeUnready(lostId);
}
for(ResourceId lostId : lost) {
auto iterModel = Models.find(lostId);
if(iterModel == Models.end())
continue;
Models.erase(iterModel);
}
std::unordered_map<std::string, ResourceId> modelKeyToId; for(const AssetsModelUpdate& entry : entries) {
if(modelAssets) { const AssetsModel key = entry.Id;
modelKeyToId.reserve(modelAssets->size());
for(const auto& [id, entry] : *modelAssets) {
modelKeyToId.emplace(entry.Domain + ':' + entry.Key, id);
}
}
for(const auto& [key, resource, deps] : newOrChanged) {
result.push_back(key); result.push_back(key);
makeUnready(key); makeUnready(key);
ModelObject model; ModelObject model;
std::string type = "unknown"; const HeadlessModel& hm = entry.Model;
std::optional<AssetsManager::ParsedHeader> header; const HeadlessModel::Header& header = entry.Header;
if(deps && !deps->empty())
header = AssetsManager::parseHeader(EnumAssets::Model, *deps);
const std::vector<uint32_t>* textureDeps = header ? &header->TextureDeps : nullptr;
auto remapTextureId = [&](uint32_t placeholder) -> uint32_t {
if(!textureDeps || placeholder >= textureDeps->size())
return 0;
return (*textureDeps)[placeholder];
};
auto remapPipeline = [&](TexturePipeline pipe) {
if(textureDeps) {
for(auto& texId : pipe.BinTextures)
texId = remapTextureId(texId);
if(!pipe.Pipeline.empty()) {
std::vector<uint8_t> code;
code.resize(pipe.Pipeline.size());
std::memcpy(code.data(), pipe.Pipeline.data(), code.size());
TexturePipelineProgram::remapTexIds(code, *textureDeps, nullptr);
pipe.Pipeline.resize(code.size());
std::memcpy(pipe.Pipeline.data(), code.data(), code.size());
}
}
return pipe;
};
try { try {
std::u8string_view data((const char8_t*) resource.data(), resource.size()); model.TextureMap.clear();
if(data.starts_with((const char8_t*) "bm")) { model.TextureMap.reserve(hm.Textures.size());
type = "InternalBinary"; for(const auto& [tkey, id] : hm.Textures) {
// Компилированная модель внутреннего формата TexturePipeline pipe;
LV::PreparedModel pm((std::u8string) data); if(id < header.TexturePipelines.size()) {
model.TextureMap.clear(); pipe.Pipeline = header.TexturePipelines[id];
model.TextureMap.reserve(pm.CompiledTextures.size()); } else {
for(auto& [tkey, pipe] : pm.CompiledTextures) LOG.warn() << "Model texture pipeline id out of range: model=" << key
model.TextureMap.emplace(tkey, remapPipeline(std::move(pipe))); << " local=" << id
model.TextureKeys = {}; << " pipelines=" << header.TexturePipelines.size();
pipe.BinTextures.push_back(id);
}
model.TextureMap.emplace(tkey, std::move(pipe));
}
model.TextureKeys = {};
for(const PreparedModel::Cuboid& cb : pm.Cuboids) { for(const HeadlessModel::Cuboid& cb : hm.Cuboids) {
glm::vec3 min = glm::min(cb.From, cb.To), max = glm::max(cb.From, cb.To); glm::vec3 min = glm::min(cb.From, cb.To), max = glm::max(cb.From, cb.To);
for(const auto& [face, params] : cb.Faces) { for(const auto& [face, params] : cb.Faces) {
glm::vec2 from_uv = {params.UV[0], params.UV[1]}, to_uv = {params.UV[2], params.UV[3]}; glm::vec2 from_uv = {params.UV[0], params.UV[1]}, to_uv = {params.UV[2], params.UV[3]};
uint32_t texId; uint32_t texId;
{ {
auto iter = std::find(model.TextureKeys.begin(), model.TextureKeys.end(), params.Texture); auto iter = std::find(model.TextureKeys.begin(), model.TextureKeys.end(), params.Texture);
if(iter == model.TextureKeys.end()) { if(iter == model.TextureKeys.end()) {
texId = model.TextureKeys.size(); texId = model.TextureKeys.size();
model.TextureKeys.push_back(params.Texture); model.TextureKeys.push_back(params.Texture);
} else { } else {
texId = iter-model.TextureKeys.begin(); texId = iter-model.TextureKeys.begin();
}
} }
}
std::vector<Vertex> v; std::vector<Vertex> v;
auto addQuad = [&](const glm::vec3& p0, auto addQuad = [&](const glm::vec3& p0,
const glm::vec3& p1, const glm::vec3& p1,
@@ -261,19 +224,23 @@ public:
} }
cb.Trs.apply(v); cb.Trs.apply(v);
model.Vertecies[params.Cullface].append_range(v); const EnumFace cullKey = (params.Cullface == EnumFace::None) ? face : params.Cullface;
model.Vertecies[cullKey].append_range(v);
} }
} }
if(!pm.SubModels.empty() && modelAssets) { if(!hm.SubModels.empty()) {
model.Depends.reserve(pm.SubModels.size()); model.Depends.reserve(hm.SubModels.size());
for(const auto& sub : pm.SubModels) { for(const auto& sub : hm.SubModels) {
auto iter = modelKeyToId.find(sub.Domain + ':' + sub.Key); if(sub.Id >= header.Models.size()) {
if(iter == modelKeyToId.end()) LOG.warn() << "Model sub-model id out of range: model=" << key
continue; << " local=" << sub.Id
model.Depends.emplace_back(iter->second, Transformations{}); << " deps=" << header.Models.size();
continue;
} }
model.Depends.emplace_back(header.Models[sub.Id], Transformations{});
} }
}
// struct Face { // struct Face {
// int TintIndex = -1; // int TintIndex = -1;
@@ -282,21 +249,21 @@ public:
// std::vector<Transformation> Transformations; // std::vector<Transformation> Transformations;
} else if(data.starts_with((const char8_t*) "glTF")) {
type = "glb";
} else if(data.starts_with((const char8_t*) "bgl")) {
type = "InternalGLTF";
} else if(data.starts_with((const char8_t*) "{")) {
type = "InternalJson или glTF";
// Модель внутреннего формата или glTF
}
} catch(const std::exception& exc) { } catch(const std::exception& exc) {
LOG.warn() << "Не удалось распарсить модель " << type << ":\n\t" << exc.what(); LOG.warn() << "Не удалось собрать модель:\n\t" << exc.what();
continue; continue;
} }
{
size_t vertexCount = 0;
for(const auto& [_, verts] : model.Vertecies)
vertexCount += verts.size();
LOG.debug() << "Model loaded id=" << key
<< " verts=" << vertexCount
<< " texKeys=" << model.TextureKeys.size()
<< " pipelines=" << header.TexturePipelines.size();
}
Models.insert_or_assign(key, std::move(model)); Models.insert_or_assign(key, std::move(model));
} }
@@ -449,7 +416,9 @@ class TextureProvider {
public: public:
struct TextureUpdate { struct TextureUpdate {
AssetsTexture Id = 0; AssetsTexture Id = 0;
Resource Res; uint16_t Width = 0;
uint16_t Height = 0;
std::vector<uint32_t> Pixels;
std::string Domain; std::string Domain;
std::string Key; std::string Key;
}; };
@@ -589,56 +558,35 @@ public:
} }
// Применяет изменения, возвращая все затронутые модели // Применяет изменения, возвращая все затронутые модели
std::vector<AssetsTexture> onTexturesChanges(std::vector<TextureUpdate> newOrChanged, std::vector<AssetsTexture> lost) { std::vector<AssetsTexture> onTexturesChanges(std::vector<AssetsTextureUpdate> entries) {
std::lock_guard lock(Mutex); std::lock_guard lock(Mutex);
std::vector<AssetsTexture> result; std::vector<AssetsTexture> result;
for(const auto& update : newOrChanged) { for(auto& entry : entries) {
const AssetsTexture key = update.Id; const AssetsTexture key = entry.Id;
const Resource& res = update.Res;
result.push_back(key); result.push_back(key);
iResource sres((const uint8_t*) res.data(), res.size()); if(entry.Width == 0 || entry.Height == 0 || entry.Pixels.empty())
iBinaryStream stream = sres.makeStream(); continue;
png::image<png::rgba_pixel> img(stream.Stream);
uint32_t width = img.get_width();
uint32_t height = img.get_height();
std::vector<uint32_t> pixels;
pixels.resize(width*height);
for(uint32_t y = 0; y < height; y++) {
const auto& row = img.get_pixbuf().operator [](y);
for(uint32_t x = 0; x < width; x++) {
const auto& px = row[x];
uint32_t rgba = (uint32_t(px.alpha) << 24)
| (uint32_t(px.red) << 16)
| (uint32_t(px.green) << 8)
| uint32_t(px.blue);
pixels[x + y * width] = rgba;
}
}
Atlas->updateTexture(key, StoredTexture( Atlas->updateTexture(key, StoredTexture(
static_cast<uint16_t>(width), entry.Width,
static_cast<uint16_t>(height), entry.Height,
std::move(pixels) std::move(entry.Pixels)
)); ));
if(auto anim = getDefaultAnimation(update.Key, width, height)) { bool animated = false;
AnimatedSources[key] = *anim;
} else {
AnimatedSources.erase(key);
}
NeedsUpload = true;
}
for(AssetsTexture key : lost) {
result.push_back(key);
Atlas->freeTexture(key);
AnimatedSources.erase(key); AnimatedSources.erase(key);
NeedsUpload = true; NeedsUpload = true;
static std::atomic<uint32_t> debugTextureLogCount = 0;
uint32_t idx = debugTextureLogCount.fetch_add(1);
if(idx < 128) {
LOG.debug() << "Texture loaded id=" << key
<< " size=" << entry.Width << 'x' << entry.Height
<< " animated=" << (animated ? 1 : 0);
}
} }
std::sort(result.begin(), result.end()); std::sort(result.begin(), result.end());
@@ -854,54 +802,16 @@ public:
{} {}
// Применяет изменения, возвращает изменённые описания состояний // Применяет изменения, возвращает изменённые описания состояний
std::vector<AssetsNodestate> onNodestateChanges(std::vector<std::tuple<AssetsNodestate, Resource, const std::vector<uint8_t>*>> newOrChanged, std::vector<AssetsNodestate> lost, std::vector<AssetsModel> changedModels) { std::vector<AssetsNodestate> onNodestateChanges(std::vector<AssetsNodestateUpdate> newOrChanged, std::vector<AssetsModel> changedModels) {
std::vector<AssetsNodestate> result; std::vector<AssetsNodestate> result;
for(ResourceId lostId : lost) {
auto iterNodestate = Nodestates.find(lostId);
if(iterNodestate == Nodestates.end())
continue;
result.push_back(lostId);
Nodestates.erase(iterNodestate);
}
for(const auto& [key, resource, deps] : newOrChanged) { for(const AssetsNodestateUpdate& entry : newOrChanged) {
const AssetsNodestate key = entry.Id;
result.push_back(key); result.push_back(key);
PreparedNodeState nodestate; PreparedNodeState nodestate;
std::string type = "unknown"; static_cast<HeadlessNodeState&>(nodestate) = entry.Nodestate;
nodestate.LocalToModel.assign(entry.Header.Models.begin(), entry.Header.Models.end());
try {
std::u8string_view data((const char8_t*) resource.data(), resource.size());
if(data.starts_with((const char8_t*) "bn")) {
type = "InternalBinary";
// Компилированный nodestate внутреннего формата
nodestate = PreparedNodeState(data);
} else if(data.starts_with((const char8_t*) "{")) {
type = "InternalJson";
// nodestate в json формате
} else {
type = "InternalBinaryLegacy";
// Старый двоичный формат без заголовка "bn"
std::u8string patched;
patched.reserve(data.size() + 2);
patched.push_back(u8'b');
patched.push_back(u8'n');
patched.append(data);
nodestate = PreparedNodeState(patched);
}
} catch(const std::exception& exc) {
LOG.warn() << "Не удалось распарсить nodestate " << type << ":\n\t" << exc.what();
continue;
}
if(deps && !deps->empty()) {
auto header = AssetsManager::parseHeader(EnumAssets::Model, *deps);
if(header && header->Type == EnumAssets::Nodestate) {
nodestate.LocalToModel.assign(header->ModelDeps.begin(), header->ModelDeps.end());
}
}
Nodestates.insert_or_assign(key, std::move(nodestate)); Nodestates.insert_or_assign(key, std::move(nodestate));
if(key < 64) { if(key < 64) {
@@ -968,6 +878,11 @@ public:
auto appendModel = [&](AssetsModel modelId, const std::vector<Transformation>& transforms, std::unordered_map<EnumFace, std::vector<NodeVertexStatic>>& out) { auto appendModel = [&](AssetsModel modelId, const std::vector<Transformation>& transforms, std::unordered_map<EnumFace, std::vector<NodeVertexStatic>>& out) {
ModelProvider::Model model = MP.getModel(modelId); ModelProvider::Model model = MP.getModel(modelId);
if(model.Vertecies.empty()) {
if(MissingModelGeometryLogged.insert(modelId).second) {
LOG.warn() << "Model has no geometry id=" << modelId;
}
}
Transformations trf{transforms}; Transformations trf{transforms};
for(auto& [l, r] : model.Vertecies) { for(auto& [l, r] : model.Vertecies) {
@@ -1015,13 +930,28 @@ public:
if(const PreparedNodeState::Model* ptr = std::get_if<PreparedNodeState::Model>(&m)) { if(const PreparedNodeState::Model* ptr = std::get_if<PreparedNodeState::Model>(&m)) {
AssetsModel modelId; AssetsModel modelId;
if(resolveModelId(ptr->Id, modelId)) if(resolveModelId(ptr->Id, modelId)) {
appendModel(modelId, ptr->Transforms, out); appendModel(modelId, ptr->Transforms, out);
} else {
uint64_t missKey = (uint64_t(id) << 32) | ptr->Id;
if(MissingLocalModelMapLogged.insert(missKey).second) {
LOG.warn() << "Missing model mapping nodestate=" << id
<< " local=" << ptr->Id
<< " locals=" << nodestate.LocalToModel.size();
}
}
} else if(const PreparedNodeState::VectorModel* ptr = std::get_if<PreparedNodeState::VectorModel>(&m)) { } else if(const PreparedNodeState::VectorModel* ptr = std::get_if<PreparedNodeState::VectorModel>(&m)) {
for(const auto& sub : ptr->Models) { for(const auto& sub : ptr->Models) {
AssetsModel modelId; AssetsModel modelId;
if(!resolveModelId(sub.Id, modelId)) if(!resolveModelId(sub.Id, modelId)) {
uint64_t missKey = (uint64_t(id) << 32) | sub.Id;
if(MissingLocalModelMapLogged.insert(missKey).second) {
LOG.warn() << "Missing model mapping nodestate=" << id
<< " local=" << sub.Id
<< " locals=" << nodestate.LocalToModel.size();
}
continue; continue;
}
std::vector<Transformation> transforms = sub.Transforms; std::vector<Transformation> transforms = sub.Transforms;
transforms.insert(transforms.end(), ptr->Transforms.begin(), ptr->Transforms.end()); transforms.insert(transforms.end(), ptr->Transforms.begin(), ptr->Transforms.end());
@@ -1058,6 +988,8 @@ private:
TextureProvider& TP; TextureProvider& TP;
std::unordered_map<AssetsNodestate, PreparedNodeState> Nodestates; std::unordered_map<AssetsNodestate, PreparedNodeState> Nodestates;
std::unordered_set<AssetsNodestate> MissingNodestateLogged; std::unordered_set<AssetsNodestate> MissingNodestateLogged;
std::unordered_set<uint64_t> MissingLocalModelMapLogged;
std::unordered_set<AssetsModel> MissingModelGeometryLogged;
std::unordered_set<uint64_t> EmptyRouteLogged; std::unordered_set<uint64_t> EmptyRouteLogged;
}; };
@@ -1167,7 +1099,7 @@ public:
assert(vkInst); assert(vkInst);
assert(serverSession); assert(serverSession);
CMG.changeThreadsCount(1); CMG.changeThreadsCount(3);
} }
~ChunkPreparator() { ~ChunkPreparator() {
@@ -1279,10 +1211,10 @@ class VulkanRenderSession : public IRenderSession {
glm::vec3 X64Offset_f, X64Delta; // Смещение мира относительно игрока в матрице вида (0 -> 64) glm::vec3 X64Offset_f, X64Delta; // Смещение мира относительно игрока в матрице вида (0 -> 64)
glm::quat Quat; glm::quat Quat;
ChunkPreparator CP;
ModelProvider MP;
std::unique_ptr<TextureProvider> TP; std::unique_ptr<TextureProvider> TP;
ModelProvider MP;
std::unique_ptr<NodestateProvider> NSP; std::unique_ptr<NodestateProvider> NSP;
ChunkPreparator CP;
AtlasImage LightDummy; AtlasImage LightDummy;
Buffer TestQuad; Buffer TestQuad;
@@ -1336,7 +1268,7 @@ public:
virtual void prepareTickSync() override; virtual void prepareTickSync() override;
virtual void pushStageTickSync() override; virtual void pushStageTickSync() override;
virtual void tickSync(const TickSyncData& data) override; virtual void tickSync(TickSyncData& data) override;
virtual void setCameraPos(WorldId_t worldId, Pos::Object pos, glm::quat quat) override; virtual void setCameraPos(WorldId_t worldId, Pos::Object pos, glm::quat quat) override;

View File

@@ -824,19 +824,121 @@ std::u8string unCompressLinear(std::u8string_view data) {
return *(std::u8string*) &outString; return *(std::u8string*) &outString;
} }
Hash_t ResourceFile::calcHash(const char8_t* data, size_t size) {
return sha2::sha256((const uint8_t*) data, size);
}
uint16_t HeadlessNodeState::Header::addModel(AssetsModel id) {
auto iter = std::find(Models.begin(), Models.end(), id);
if(iter == Models.end()) {
Models.push_back(id);
return Models.size() - 1;
}
return iter - Models.begin();
}
void HeadlessNodeState::Header::load(std::u8string_view data) {
Models.clear();
if(data.empty())
return;
if(data.size() % sizeof(AssetsModel) != 0)
MAKE_ERROR("Invalid nodestate header size");
const size_t count = data.size() / sizeof(AssetsModel);
Models.resize(count);
std::memcpy(Models.data(), data.data(), data.size());
}
ResourceHeader HeadlessNodeState::Header::dump() const {
ResourceHeader rh;
rh.reserve(Models.size() * sizeof(AssetsModel));
for(AssetsModel id : Models) {
rh += std::u8string_view((const char8_t*) &id, sizeof(AssetsModel));
}
return rh;
}
uint16_t HeadlessModel::Header::addModel(AssetsModel id) {
auto iter = std::find(Models.begin(), Models.end(), id);
if(iter == Models.end()) {
Models.push_back(id);
return Models.size() - 1;
}
return iter - Models.begin();
}
uint16_t HeadlessModel::Header::addTexturePipeline(std::vector<uint8_t> pipeline) {
TexturePipelines.push_back(std::move(pipeline));
return TexturePipelines.size() - 1;
}
void HeadlessModel::Header::load(std::u8string_view data) {
Models.clear();
TexturePipelines.clear();
if(data.empty())
return;
try {
TOS::ByteBuffer buffer(data.size(), reinterpret_cast<const uint8_t*>(data.data()));
auto reader = buffer.reader();
uint16_t modelCount = reader.readUInt16();
Models.reserve(modelCount);
for(uint16_t i = 0; i < modelCount; ++i)
Models.push_back(reader.readUInt32());
uint16_t texCount = reader.readUInt16();
TexturePipelines.reserve(texCount);
for(uint16_t i = 0; i < texCount; ++i) {
uint32_t size32 = reader.readUInt32();
TOS::ByteBuffer pipe;
reader.readBuffer(pipe);
if(pipe.size() != size32)
MAKE_ERROR("Invalid model header size");
TexturePipelines.emplace_back(pipe.begin(), pipe.end());
}
} catch(const std::exception&) {
MAKE_ERROR("Invalid model header");
}
}
ResourceHeader HeadlessModel::Header::dump() const {
TOS::ByteBuffer rh;
{
uint32_t fullSize = 0;
for(const auto& vector : TexturePipelines)
fullSize += vector.size();
rh.reserve(2 + Models.size() * sizeof(AssetsModel) + 2 + 4 * TexturePipelines.size() + fullSize);
}
TOS::ByteBuffer::Writer wr;
wr << uint16_t(Models.size());
for(AssetsModel id : Models)
wr << id;
wr << uint16_t(TexturePipelines.size());
for(const auto& pipe : TexturePipelines) {
wr << uint32_t(pipe.size());
wr << pipe;
}
TOS::ByteBuffer buff = wr.complite();
return std::u8string((const char8_t*) buff.data(), buff.size());
}
ResourceHeader HeadlessNodeState::parse(const js::object& profile, const std::function<AssetsModel(const std::string_view model)>& modelResolver) { ResourceHeader HeadlessNodeState::parse(const js::object& profile, const std::function<AssetsModel(const std::string_view model)>& modelResolver) {
std::vector<AssetsModel> headerIds; Header header;
std::function<uint16_t(const std::string_view model)> headerResolver = std::function<uint16_t(const std::string_view model)> headerResolver =
[&](const std::string_view model) -> uint16_t { [&](const std::string_view model) -> uint16_t {
AssetsModel id = modelResolver(model); AssetsModel id = modelResolver(model);
auto iter = std::find(headerIds.begin(), headerIds.end(), id); return header.addModel(id);
if(iter == headerIds.end()) {
headerIds.push_back(id);
return headerIds.size()-1;
}
return iter-headerIds.begin();
}; };
for(auto& [condition, variability] : profile) { for(auto& [condition, variability] : profile) {
@@ -865,14 +967,7 @@ ResourceHeader HeadlessNodeState::parse(const js::object& profile, const std::fu
Routes.emplace_back(node, std::move(models)); Routes.emplace_back(node, std::move(models));
} }
ResourceHeader rh; return header.dump();
rh.reserve(headerIds.size()*sizeof(AssetsModel));
for(AssetsModel id : headerIds) {
rh += std::u8string_view((const char8_t*) &id, sizeof(AssetsModel));
}
return rh;
} }
ResourceHeader HeadlessNodeState::parse(const sol::table& profile, const std::function<AssetsModel(const std::string_view model)>& modelResolver) { ResourceHeader HeadlessNodeState::parse(const sol::table& profile, const std::function<AssetsModel(const std::string_view model)>& modelResolver) {
@@ -1490,21 +1585,14 @@ ResourceHeader HeadlessModel::parse(
const std::function<AssetsModel(const std::string_view model)>& modelResolver, const std::function<AssetsModel(const std::string_view model)>& modelResolver,
const std::function<std::vector<uint8_t>(const std::string_view texturePipelineSrc)>& textureResolver const std::function<std::vector<uint8_t>(const std::string_view texturePipelineSrc)>& textureResolver
) { ) {
std::vector<AssetsModel> headerIdsModels; Header header;
std::function<uint16_t(const std::string_view model)> headerResolverModel = std::function<uint16_t(const std::string_view model)> headerResolverModel =
[&](const std::string_view model) -> uint16_t { [&](const std::string_view model) -> uint16_t {
AssetsModel id = modelResolver(model); AssetsModel id = modelResolver(model);
auto iter = std::find(headerIdsModels.begin(), headerIdsModels.end(), id); return header.addModel(id);
if(iter == headerIdsModels.end()) {
headerIdsModels.push_back(id);
return headerIdsModels.size()-1;
}
return iter-headerIdsModels.begin();
}; };
std::vector<std::vector<uint8_t>> headerIdsTextures;
std::unordered_map<std::string, uint32_t, detail::TSVHash, detail::TSVEq> textureToLocal; std::unordered_map<std::string, uint32_t, detail::TSVHash, detail::TSVEq> textureToLocal;
std::function<uint16_t(const std::string_view texturePipelineSrc)> headerResolverTexture = std::function<uint16_t(const std::string_view texturePipelineSrc)> headerResolverTexture =
@@ -1514,9 +1602,8 @@ ResourceHeader HeadlessModel::parse(
return iter->second; return iter->second;
} }
std::vector<uint8_t> program = textureResolver(texturePipelineSrc); uint16_t id = header.addTexturePipeline(textureResolver(texturePipelineSrc));
headerIdsTextures.push_back(program); textureToLocal[(std::string) texturePipelineSrc] = id;
uint16_t id = textureToLocal[(std::string) texturePipelineSrc] = headerIdsTextures.size()-1;
return id; return id;
}; };
@@ -1741,50 +1828,10 @@ ResourceHeader HeadlessModel::parse(
} }
} }
if(boost::system::result<const js::value&> subModels_val = profile.try_at("sub_models")) {
const js::array& subModels = subModels_val->as_array();
for(const js::value& sub_val : subModels) {
SubModel result;
if(auto path = sub_val.try_as_string()) {
result.Id = headerResolverModel(path.value());
} else {
const js::object& sub = sub_val.as_object();
result.Id = headerResolverModel(sub.at("path").as_string());
if(boost::system::result<const js::value&> scene_val = profile.try_at("scene"))
result.Scene = scene_val->to_number<uint16_t>();
}
SubModels.emplace_back(std::move(result));
}
}
// Заголовок // Заголовок
TOS::ByteBuffer rh; TOS::ByteBuffer rh;
{ return header.dump();
uint32_t fullSize = 0;
for(const auto& vector : headerIdsTextures)
fullSize += vector.size();
rh.reserve(2+headerIdsModels.size()*sizeof(AssetsModel)+2+(4)*headerIdsTextures.size()+fullSize);
}
TOS::ByteBuffer::Writer wr;
wr << uint16_t(headerIdsModels.size());
for(AssetsModel id : headerIdsModels)
wr << id;
wr << uint16_t(headerIdsTextures.size());
for(const auto& pipe : headerIdsTextures) {
wr << uint32_t(pipe.size());
wr << pipe;
}
TOS::ByteBuffer buff = wr.complite();
return std::u8string((const char8_t*) buff.data(), buff.size());
} }
ResourceHeader HeadlessModel::parse( ResourceHeader HeadlessModel::parse(
@@ -2004,37 +2051,6 @@ std::u8string HeadlessModel::dump() const {
return result.complite(); return result.complite();
} }
PreparedGLTF::PreparedGLTF(const std::string_view modid, const js::object& gltf) {
// gltf
// Сцена по умолчанию
// Сцены -> Ноды
// Ноды -> Ноды, меши, матрицы, translation, rotation
// Меши -> Примитивы
// Примитивы -> Материал, вершинные данные
// Материалы -> текстуры
// Текстуры
// Буферы
}
PreparedGLTF::PreparedGLTF(const std::string_view modid, Resource glb) {
}
PreparedGLTF::PreparedGLTF(std::u8string_view data) {
// lr.checkUnreaded();
}
std::u8string PreparedGLTF::dump() const {
std::unreachable();
}
void PreparedGLTF::load(std::u8string_view data) {
}
struct Resource::InlineMMap { struct Resource::InlineMMap {
boost::interprocess::file_mapping MMap; boost::interprocess::file_mapping MMap;
boost::interprocess::mapped_region Region; boost::interprocess::mapped_region Region;

View File

@@ -553,6 +553,19 @@ inline std::pair<std::string, std::string> parseDomainKey(const std::string& val
} }
} }
struct ResourceFile {
using Hash_t = std::array<uint8_t, 32>;
Hash_t Hash;
std::u8string Data;
static Hash_t calcHash(const char8_t* data, size_t size);
void calcHash() {
Hash = calcHash(Data.data(), Data.size());
}
};
struct NodestateEntry { struct NodestateEntry {
std::string Name; std::string Name;
int Variability = 0; // Количество возможный значений состояния int Variability = 0; // Количество возможный значений состояния
@@ -662,6 +675,14 @@ struct HeadlessNodeState {
std::vector<Transformation> Transforms; std::vector<Transformation> Transforms;
}; };
struct Header {
std::vector<AssetsModel> Models;
uint16_t addModel(AssetsModel id);
void load(std::u8string_view data);
ResourceHeader dump() const;
};
// Ноды выражений // Ноды выражений
std::vector<Node> Nodes; std::vector<Node> Nodes;
// Условия -> вариации модели + веса // Условия -> вариации модели + веса
@@ -898,6 +919,16 @@ struct HeadlessModel {
std::optional<EnumGuiLight> GuiLight = EnumGuiLight::Default; std::optional<EnumGuiLight> GuiLight = EnumGuiLight::Default;
std::optional<bool> AmbientOcclusion = false; std::optional<bool> AmbientOcclusion = false;
struct Header {
std::vector<AssetsModel> Models;
std::vector<std::vector<uint8_t>> TexturePipelines;
uint16_t addModel(AssetsModel id);
uint16_t addTexturePipeline(std::vector<uint8_t> pipeline);
void load(std::u8string_view data);
ResourceHeader dump() const;
};
struct FullTransformation { struct FullTransformation {
glm::vec3 glm::vec3
@@ -982,37 +1013,6 @@ struct TexturePipeline {
} }
}; };
struct PreparedModel {
struct SubModel {
std::string Domain;
std::string Key;
};
using Cuboid = HeadlessModel::Cuboid;
std::unordered_map<std::string, TexturePipeline> CompiledTextures;
std::vector<Cuboid> Cuboids;
std::vector<SubModel> SubModels;
PreparedModel() = default;
PreparedModel(const std::u8string& data) { load(data); }
PreparedModel(std::u8string_view data) { load(data); }
void load(std::u8string_view data) {
HeadlessModel model;
model.load(data);
Cuboids = model.Cuboids;
CompiledTextures.clear();
CompiledTextures.reserve(model.Textures.size());
for(const auto& [key, id] : model.Textures) {
TexturePipeline pipe;
pipe.BinTextures.push_back(id);
CompiledTextures.emplace(key, std::move(pipe));
}
}
};
struct PreparedNodeState : public HeadlessNodeState { struct PreparedNodeState : public HeadlessNodeState {
using HeadlessNodeState::Model; using HeadlessNodeState::Model;
using HeadlessNodeState::VectorModel; using HeadlessNodeState::VectorModel;
@@ -1024,30 +1024,6 @@ struct PreparedNodeState : public HeadlessNodeState {
PreparedNodeState(const std::u8string& data) { load(data); } PreparedNodeState(const std::u8string& data) { load(data); }
}; };
struct PreparedGLTF {
std::vector<std::string> TextureKey;
std::unordered_map<std::string, uint16_t> Textures;
std::vector<Vertex> Vertices;
PreparedGLTF(const std::string_view modid, const js::object& gltf);
PreparedGLTF(const std::string_view modid, Resource glb);
PreparedGLTF(std::u8string_view data);
PreparedGLTF() = default;
PreparedGLTF(const PreparedGLTF&) = default;
PreparedGLTF(PreparedGLTF&&) = default;
PreparedGLTF& operator=(const PreparedGLTF&) = default;
PreparedGLTF& operator=(PreparedGLTF&&) = default;
// Пишет в сжатый двоичный формат
std::u8string dump() const;
private:
void load(std::u8string_view data);
};
enum struct TexturePipelineCMD : uint8_t { enum struct TexturePipelineCMD : uint8_t {
Texture, // Указание текстуры Texture, // Указание текстуры
Combine, // Комбинирование Combine, // Комбинирование

View File

@@ -1,10 +1,17 @@
#include "AssetsPreloader.hpp" #include "AssetsPreloader.hpp"
#include "Common/Abstract.hpp"
#include "Common/TexturePipelineProgram.hpp"
#include "sha2.hpp"
#include <atomic>
#include <filesystem>
#include <fstream> #include <fstream>
#include <unordered_set> #include <unordered_set>
#include <utility> #include <utility>
namespace LV { namespace LV {
static TOS::Logger LOG = "AssetsPreloader";
static ResourceFile readFileBytes(const fs::path& path) { static ResourceFile readFileBytes(const fs::path& path) {
std::ifstream file(path, std::ios::binary); std::ifstream file(path, std::ios::binary);
if(!file) if(!file)
@@ -39,30 +46,53 @@ static std::u8string readOptionalMeta(const fs::path& path) {
} }
AssetsPreloader::AssetsPreloader() { AssetsPreloader::AssetsPreloader() {
std::fill(NextId.begin(), NextId.end(), 1); for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); type++) {
std::fill(LastSendId.begin(), LastSendId.end(), 1); ResourceLinks[type].emplace_back(
ResourceFile::Hash_t{0},
ResourceHeader(),
fs::file_time_type(),
fs::path{""},
false
);
}
} }
AssetsPreloader::Out_reloadResources AssetsPreloader::reloadResources(const AssetsRegister& instances, ReloadStatus* status) { AssetsPreloader::Out_checkAndPrepareResourcesUpdate AssetsPreloader::checkAndPrepareResourcesUpdate(
const AssetsRegister& instances,
const std::function<ResourceId(EnumAssets type, std::string_view domain, std::string_view key)>& idResolver,
const std::function<void(std::u8string&& resource, ResourceFile::Hash_t hash, fs::path resPath)>& onNewResourceParsed,
ReloadStatus* status
) {
assert(idResolver);
#ifndef NDEBUG
bool expected = false; bool expected = false;
assert(_Reloading.compare_exchange_strong(expected, true) && "Двойной вызов reloadResources"); assert(_Reloading.compare_exchange_strong(expected, true) && "Двойной вызов reloadResources");
struct ReloadGuard { struct ReloadGuard {
std::atomic<bool>& Flag; std::atomic<bool>& Flag;
~ReloadGuard() { Flag.exchange(false); } ~ReloadGuard() { Flag.exchange(false); }
} guard{_Reloading}; } guard{_Reloading};
#endif
try { try {
ReloadStatus secondStatus; ReloadStatus secondStatus;
return _reloadResources(instances, status ? *status : secondStatus); return _checkAndPrepareResourcesUpdate(instances, idResolver, onNewResourceParsed, status ? *status : secondStatus);
} catch(const std::exception& exc) {
LOG.error() << exc.what();
assert(!"reloadResources: здесь не должно быть ошибок");
std::unreachable();
} catch(...) { } catch(...) {
assert(!"reloadResources: здесь не должно быть ошибок"); assert(!"reloadResources: здесь не должно быть ошибок");
std::unreachable(); std::unreachable();
} }
} }
AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const AssetsRegister& instances, ReloadStatus& status) { AssetsPreloader::Out_checkAndPrepareResourcesUpdate AssetsPreloader::_checkAndPrepareResourcesUpdate(
Out_reloadResources result; const AssetsRegister& instances,
const std::function<ResourceId(EnumAssets type, std::string_view domain, std::string_view key)>& idResolver,
const std::function<void(std::u8string&& resource, ResourceFile::Hash_t hash, fs::path resPath)>& onNewResourceParsed,
ReloadStatus& status
) {
// 1) Поиск всех ресурсов и построение конечной карты ресурсов (timestamps, path, name, size) // 1) Поиск всех ресурсов и построение конечной карты ресурсов (timestamps, path, name, size)
// Карта найденных ресурсов // Карта найденных ресурсов
std::array< std::array<
@@ -80,12 +110,12 @@ AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const Ass
static_cast<size_t>(AssetType::MAX_ENUM) static_cast<size_t>(AssetType::MAX_ENUM)
> resourcesFirstStage; > resourcesFirstStage;
for (const fs::path& instance : instances.Assets) { for(const fs::path& instance : instances.Assets) {
try { try {
if (fs::is_regular_file(instance)) { if(fs::is_regular_file(instance)) {
// Может архив // Может архив
/// TODO: пока не поддерживается /// TODO: пока не поддерживается
} else if (fs::is_directory(instance)) { } else if(fs::is_directory(instance)) {
// Директория // Директория
fs::path assetsRoot = instance; fs::path assetsRoot = instance;
fs::path assetsCandidate = instance / "assets"; fs::path assetsCandidate = instance / "assets";
@@ -115,20 +145,20 @@ AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const Ass
>& firstStage = resourcesFirstStage[static_cast<size_t>(assetType)][domain]; >& firstStage = resourcesFirstStage[static_cast<size_t>(assetType)][domain];
// Исследуем все ресурсы одного типа // Исследуем все ресурсы одного типа
for (auto begin = fs::recursive_directory_iterator(assetPath), end = fs::recursive_directory_iterator(); begin != end; begin++) { for(auto begin = fs::recursive_directory_iterator(assetPath), end = fs::recursive_directory_iterator(); begin != end; begin++) {
if (begin->is_directory()) if(begin->is_directory())
continue; continue;
fs::path file = begin->path(); fs::path file = begin->path();
if (assetType == AssetType::Texture && file.extension() == ".meta") if(assetType == AssetType::Texture && file.extension() == ".meta")
continue; continue;
std::string key = fs::relative(file, assetPath).string(); std::string key = fs::relative(file, assetPath).generic_string();
if (firstStage.contains(key)) if(firstStage.contains(key))
continue; continue;
fs::file_time_type timestamp = fs::last_write_time(file); fs::file_time_type timestamp = fs::last_write_time(file);
if (assetType == AssetType::Texture) { if(assetType == AssetType::Texture) {
fs::path metaPath = file; fs::path metaPath = file;
metaPath += ".meta"; metaPath += ".meta";
if (fs::exists(metaPath) && fs::is_regular_file(metaPath)) { if (fs::exists(metaPath) && fs::is_regular_file(metaPath)) {
@@ -141,7 +171,8 @@ AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const Ass
// Работаем с ресурсом // Работаем с ресурсом
firstStage[key] = ResourceFindInfo{ firstStage[key] = ResourceFindInfo{
.Path = file, .Path = file,
.Timestamp = timestamp .Timestamp = timestamp,
.Id = idResolver(assetType, domain, key)
}; };
} }
} }
@@ -151,7 +182,6 @@ AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const Ass
} }
} catch (const std::exception& exc) { } catch (const std::exception& exc) {
/// TODO: Логгировать в статусе /// TODO: Логгировать в статусе
} }
} }
@@ -165,21 +195,21 @@ AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const Ass
= [&](const std::string_view model) -> uint32_t = [&](const std::string_view model) -> uint32_t
{ {
auto [mDomain, mKey] = parseDomainKey(model, domain); auto [mDomain, mKey] = parseDomainKey(model, domain);
return getId(AssetType::Model, mDomain, mKey); return idResolver(AssetType::Model, mDomain, mKey);
}; };
std::function<std::optional<uint32_t>(std::string_view)> textureIdResolver std::function<std::optional<uint32_t>(std::string_view)> textureIdResolver
= [&](std::string_view texture) -> std::optional<uint32_t> = [&](std::string_view texture) -> std::optional<uint32_t>
{ {
auto [mDomain, mKey] = parseDomainKey(texture, domain); auto [mDomain, mKey] = parseDomainKey(texture, domain);
return getId(AssetType::Texture, mDomain, mKey); return idResolver(AssetType::Texture, mDomain, mKey);
}; };
std::function<std::vector<uint8_t>(const std::string_view)> textureResolver std::function<std::vector<uint8_t>(const std::string_view)> textureResolver
= [&](const std::string_view texturePipelineSrc) -> std::vector<uint8_t> = [&](const std::string_view texturePipelineSrc) -> std::vector<uint8_t>
{ {
TexturePipelineProgram tpp; TexturePipelineProgram tpp;
bool flag = tpp.compile((std::string) texturePipelineSrc); bool flag = tpp.compile(texturePipelineSrc);
if(!flag) if(!flag)
return {}; return {};
@@ -195,8 +225,8 @@ AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const Ass
HeadlessNodeState hns; HeadlessNodeState hns;
out.Header = hns.parse(obj, modelResolver); out.Header = hns.parse(obj, modelResolver);
out.Resource = std::make_shared<std::u8string>(hns.dump()); out.Resource = hns.dump();
out.Hash = sha2::sha256((const uint8_t*) out.Resource->data(), out.Resource->size()); out.Hash = sha2::sha256((const uint8_t*) out.Resource.data(), out.Resource.size());
} else if (type == AssetType::Model) { } else if (type == AssetType::Model) {
const std::string ext = info.Path.extension().string(); const std::string ext = info.Path.extension().string();
if (ext == ".json") { if (ext == ".json") {
@@ -206,8 +236,8 @@ AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const Ass
HeadlessModel hm; HeadlessModel hm;
out.Header = hm.parse(obj, modelResolver, textureResolver); out.Header = hm.parse(obj, modelResolver, textureResolver);
out.Resource = std::make_shared<std::u8string>(hm.dump()); out.Resource = hm.dump();
out.Hash = sha2::sha256((const uint8_t*) out.Resource->data(), out.Resource->size()); out.Hash = sha2::sha256((const uint8_t*) out.Resource.data(), out.Resource.size());
// } else if (ext == ".gltf" || ext == ".glb") { // } else if (ext == ".gltf" || ext == ".glb") {
// /// TODO: добавить поддержку gltf // /// TODO: добавить поддержку gltf
// ResourceFile file = readFileBytes(info.Path); // ResourceFile file = readFileBytes(info.Path);
@@ -218,239 +248,152 @@ AssetsPreloader::Out_reloadResources AssetsPreloader::_reloadResources(const Ass
} }
} else if (type == AssetType::Texture) { } else if (type == AssetType::Texture) {
ResourceFile file = readFileBytes(info.Path); ResourceFile file = readFileBytes(info.Path);
out.Resource = std::make_shared<std::u8string>(std::move(file.Data)); out.Resource = std::move(file.Data);
out.Hash = file.Hash; out.Hash = file.Hash;
out.Header = readOptionalMeta(info.Path); out.Header = readOptionalMeta(info.Path);
} else { } else {
ResourceFile file = readFileBytes(info.Path); ResourceFile file = readFileBytes(info.Path);
out.Resource = std::make_shared<std::u8string>(std::move(file.Data)); out.Resource = std::move(file.Data);
out.Hash = file.Hash; out.Hash = file.Hash;
} }
out.Id = getId(type, domain, key); out.Id = idResolver(type, domain, key);
return out; return out;
}; };
// 2) Определяем какие ресурсы изменились (новый timestamp) или новые ресурсы
Out_checkAndPrepareResourcesUpdate result;
// Собираем идентификаторы, чтобы потом определить какие ресурсы пропали
std::array<
std::unordered_set<ResourceId>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
> uniqueExists;
// 2) Обрабатываться будут только изменённые (новый timestamp) или новые ресурсы
// Определяем каких ресурсов не стало
for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); ++type) { for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); ++type) {
auto& tableResourcesFirstStage = resourcesFirstStage[type]; auto& uniqueExistsTypes = uniqueExists[type];
for(const auto& [id, resource] : MediaResources[type]) { const auto& resourceLinksTyped = ResourceLinks[type];
if(tableResourcesFirstStage.empty()) { result.MaxNewSize[type] = resourceLinksTyped.size();
result.Lost[type][resource.Domain].push_back(resource.Key);
continue;
}
auto iterDomain = tableResourcesFirstStage.find(resource.Domain); {
if(iterDomain == tableResourcesFirstStage.end()) { size_t allIds = 0;
result.Lost[type][resource.Domain].push_back(resource.Key); for(const auto& [domain, keys] : resourcesFirstStage[type])
continue; allIds += keys.size();
}
if(!iterDomain->second.contains(resource.Key)) { uniqueExistsTypes.reserve(allIds);
result.Lost[type][resource.Domain].push_back(resource.Key);
}
} }
}
// Определение новых или изменённых ресурсов for(const auto& [domain, keys] : resourcesFirstStage[type]) {
for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); ++type) { for(const auto& [key, res] : keys) {
for(const auto& [domain, table] : resourcesFirstStage[type]) { uniqueExistsTypes.insert(res.Id);
auto iterTableDomain = DKToId[type].find(domain);
if(iterTableDomain == DKToId[type].end()) { if(res.Id >= resourceLinksTyped.size() || !resourceLinksTyped[res.Id].IsExist)
// Домен неизвестен движку, все ресурсы в нём новые { // Если идентификатора нет в таблице или ресурс не привязан
for(const auto& [key, info] : table) { PendingResource resource = buildResource(static_cast<AssetType>(type), domain, key, res);
PendingResource resource = buildResource(static_cast<AssetType>(type), domain, key, info); if(onNewResourceParsed)
result.NewOrChange[type][domain].push_back(std::move(resource)); onNewResourceParsed(std::move(resource.Resource), resource.Hash, res.Path);
} result.HashToPathNew[resource.Hash].push_back(res.Path);
} else {
for(const auto& [key, info] : table) { if(res.Id >= result.MaxNewSize[type])
bool needsUpdate = true; result.MaxNewSize[type] = res.Id+1;
if(auto iterKey = iterTableDomain->second.find(key); iterKey != iterTableDomain->second.end()) {
// Идентификатор найден result.ResourceUpdates[type].emplace_back(res.Id, resource.Hash, std::move(resource.Header), resource.Timestamp, res.Path);
auto iterRes = MediaResources[type].find(iterKey->second); } else if(resourceLinksTyped[res.Id].Path != res.Path
// Если нашли ресурс по идентификатору и время изменения не поменялось, то он не новый и не изменился || resourceLinksTyped[res.Id].LastWrite != res.Timestamp
if(iterRes != MediaResources[type].end() && iterRes->second.Timestamp == info.Timestamp) ) { // Если ресурс теперь берётся с другого места или изменилось время изменения файла
needsUpdate = false; const auto& lastResource = resourceLinksTyped[res.Id];
PendingResource resource = buildResource(static_cast<AssetType>(type), domain, key, res);
if(lastResource.Hash != resource.Hash) {
// Хэш изменился
// Сообщаем о новом ресурсе
if(onNewResourceParsed)
onNewResourceParsed(std::move(resource.Resource), resource.Hash, res.Path);
// Старый хэш более не доступен по этому расположению.
result.HashToPathLost[lastResource.Hash].push_back(resourceLinksTyped[res.Id].Path);
// Новый хеш стал доступен по этому расположению.
result.HashToPathNew[resource.Hash].push_back(res.Path);
} else if(resourceLinksTyped[res.Id].Path != res.Path) {
// Изменился конечный путь.
// Хэш более не доступен по этому расположению.
result.HashToPathLost[resource.Hash].push_back(resourceLinksTyped[res.Id].Path);
// Хеш теперь доступен по этому расположению.
result.HashToPathNew[resource.Hash].push_back(res.Path);
} else {
// Ресурс без заголовка никак не изменился.
} }
if(!needsUpdate) // Чтобы там не поменялось, мог поменятся заголовок. Уведомляем о новой привязке.
continue; result.ResourceUpdates[type].emplace_back(res.Id, resource.Hash, std::move(resource.Header), resource.Timestamp, res.Path);
} else {
PendingResource resource = buildResource(static_cast<AssetType>(type), domain, key, info); // Ресурс не изменился
result.NewOrChange[(int) type][domain].push_back(std::move(resource));
} }
} }
} }
} }
return result; // 3) Определяем какие ресурсы пропали
}
AssetsPreloader::Out_applyResourceChange AssetsPreloader::applyResourceChange(const Out_reloadResources& orr) {
Out_applyResourceChange result;
// Удаляем ресурсы
/*
Удаляются только ресурсы, при этом за ними остаётся бронь на идентификатор
Уже скомпилированные зависимости к ресурсам не будут
перекомпилироваться для смены идентификатора.
Если нужный ресурс появится, то привязка останется.
Новые клиенты не получат ресурс которого нет,
но он может использоваться
*/
for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); type++) {
for(const auto& [domain, keys] : orr.Lost[type]) {
auto iterDomain = DKToId[type].find(domain);
// Если уже было решено, что ресурсы были, и стали потерянными, то так и должно быть
assert(iterDomain != DKToId[type].end());
for(const auto& key : keys) {
auto iterKey = iterDomain->second.find(key);
// Ресурс был и должен быть
assert(iterKey != iterDomain->second.end());
uint32_t id = iterKey->second;
auto& resType = MediaResources[type];
auto iterRes = resType.find(id);
if(iterRes == resType.end())
continue;
// Ресурс был потерян
result.Lost[type].push_back(id);
// Hash более нам неизвестен
HashToId.erase(iterRes->second.Hash);
// Затираем ресурс
resType.erase(iterRes);
}
}
}
// Добавляем
for(int type = 0; type < (int) AssetType::MAX_ENUM; type++) {
auto& typeTable = DKToId[type];
for(const auto& [domain, resources] : orr.NewOrChange[type]) {
auto& domainTable = typeTable[domain];
for(const PendingResource& pending : resources) {
MediaResource resource {
.Domain = domain,
.Key = std::move(pending.Key),
.Timestamp = pending.Timestamp,
.Resource = std::move(pending.Resource),
.Hash = pending.Hash,
.Header = std::move(pending.Header)
};
auto& table = MediaResources[type];
// Нужно затереть старую ссылку хеша на данный ресурс
if(auto iter = table.find(pending.Id); iter != table.end())
HashToId.erase(iter->second.Hash);
// Добавили ресурс
table[pending.Id] = resource;
// Связали с хешем
HashToId[resource.Hash] = {static_cast<AssetType>(type), pending.Id};
// Осведомили о новом/изменённом ресурсе
result.NewOrChange[type].emplace_back(pending.Id, resource.Hash, std::move(resource.Header));
}
}
// Не должно быть ресурсов, которые были помечены как потерянные
#ifndef NDEBUG
std::unordered_set<uint32_t> changed;
for(const auto& [id, _, _] : result.NewOrChange[type])
changed.insert(id);
auto& lost = result.Lost[type];
for(auto iter : lost)
assert(!changed.contains(iter));
#endif
}
return result;
}
AssetsPreloader::Out_bakeId AssetsPreloader::bakeIdTables() {
#ifndef NDEBUG
assert(!DKToIdInBakingMode);
DKToIdInBakingMode = true;
struct _tempStruct {
AssetsPreloader* handler;
~_tempStruct() { handler->DKToIdInBakingMode = false; }
} _lock{this};
#endif
Out_bakeId result;
for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); ++type) { for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); ++type) {
// домен+ключ -> id const auto& resourceLinksTyped = ResourceLinks[type];
{
auto lock = NewDKToId[type].lock();
auto& dkToId = DKToId[type];
for(auto& [domain, keys] : *lock) {
// Если домен не существует, просто воткнёт новые ключи
auto [iterDomain, inserted] = dkToId.try_emplace(domain, std::move(keys));
if(!inserted) {
// Домен уже существует, сливаем новые ключи
iterDomain->second.merge(keys);
}
}
lock->clear(); size_t counter = 0;
} for(const auto& [hash, header, timestamp, path, isExist] : resourceLinksTyped) {
size_t id = counter++;
if(!isExist)
continue;
// id -> домен+ключ if(uniqueExists[type].contains(id))
{ continue;
auto lock = NewIdToDK[type].lock();
auto& idToDK = IdToDK[type]; // Ресурс потерян
result.IdToDK[type] = std::move(*lock); // Хэш более не доступен по этому расположению.
lock->clear(); result.HashToPathLost[hash].push_back(path);
idToDK.append_range(result.IdToDK[type]); result.LostLinks[type].push_back(id);
// result.LastSendId[type] = LastSendId[type];
LastSendId[type] = NextId[type];
} }
} }
return result; return result;
} }
AssetsPreloader::Out_fullSync AssetsPreloader::collectFullSync() const { AssetsPreloader::Out_applyResourcesUpdate AssetsPreloader::applyResourcesUpdate(const Out_checkAndPrepareResourcesUpdate& orr) {
Out_fullSync out; Out_applyResourcesUpdate result;
for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); ++type) { for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); ++type) {
out.IdToDK[type] = IdToDK[type]; // Затираем потерянные
} for(ResourceId id : orr.LostLinks[type]) {
assert(id < ResourceLinks[type].size());
auto& [hash, header, timestamp, path, isExist] = ResourceLinks[type][id];
hash = {0};
header = {};
timestamp = fs::file_time_type();
path.clear();
isExist = false;
for(size_t type = 0; type < static_cast<size_t>(AssetType::MAX_ENUM); ++type) { result.NewOrUpdates[type].emplace_back(id, hash, header);
for(const auto& [id, resource] : MediaResources[type]) { }
out.HashHeaders[type].push_back(BindHashHeaderInfo{
.Id = id, // Увеличиваем размер, если необходимо
.Hash = resource.Hash, if(orr.MaxNewSize[type] > ResourceLinks[type].size()) {
.Header = resource.Header ResourceLink def{
}); ResourceFile::Hash_t{0},
out.Resources.emplace_back( ResourceHeader(),
static_cast<AssetType>(type), fs::file_time_type(),
id, fs::path{""},
&resource false
); };
ResourceLinks[type].resize(orr.MaxNewSize[type], def);
}
// Обновляем / добавляем
for(auto& [id, hash, header, timestamp, path] : orr.ResourceUpdates[type]) {
ResourceLinks[type][id] = {hash, std::move(header), timestamp, std::move(path), true};
result.NewOrUpdates[type].emplace_back(id, hash, header);
} }
} }
return out; return result;
}
std::tuple<AssetsNodestate, std::vector<AssetsModel>, std::vector<AssetsTexture>>
AssetsPreloader::getNodeDependency(const std::string& domain, const std::string& key) {
(void)domain;
(void)key;
return {0, {}, {}};
} }
} }

View File

@@ -1,6 +1,5 @@
#pragma once #pragma once
#include <algorithm>
#include <array> #include <array>
#include <cstdint> #include <cstdint>
#include <cstring> #include <cstring>
@@ -12,19 +11,13 @@
#include <string_view> #include <string_view>
#include <tuple> #include <tuple>
#include <unordered_map> #include <unordered_map>
#include <utility>
#include <vector> #include <vector>
#include "Common/TexturePipelineProgram.hpp" #include "Abstract.hpp"
#include "Common/Abstract.hpp" #include "Common/Abstract.hpp"
#include "Common/Async.hpp"
#include "TOSAsync.hpp"
#include "TOSLib.hpp"
#include "sha2.hpp"
/* /*
Класс отвечает за отслеживание изменений и подгрузки медиаресурсов в указанных директориях. Класс отвечает за отслеживание изменений и подгрузки медиаресурсов в указанных директориях.
Медиаресурсы, собранные из папки assets или зарегистрированные модами. Медиаресурсы, собранные из папки assets или зарегистрированные модами.
Хранит все данные в оперативной памяти.
*/ */
static constexpr const char* EnumAssetsToDirectory(LV::EnumAssets value) { static constexpr const char* EnumAssetsToDirectory(LV::EnumAssets value) {
@@ -49,34 +42,10 @@ namespace LV {
namespace fs = std::filesystem; namespace fs = std::filesystem;
using AssetType = EnumAssets; using AssetType = EnumAssets;
struct ResourceFile {
using Hash_t = sha2::sha256_hash; // boost::uuids::detail::sha1::digest_type;
Hash_t Hash;
std::u8string Data;
void calcHash() {
Hash = sha2::sha256((const uint8_t*) Data.data(), Data.size());
}
};
class AssetsPreloader { class AssetsPreloader {
public: public:
using Ptr = std::shared_ptr<AssetsPreloader>; using Ptr = std::shared_ptr<AssetsPreloader>;
using IdTable =
std::unordered_map<
std::string, // Domain
std::unordered_map<
std::string, // Key
uint32_t, // ResourceId
detail::TSVHash,
detail::TSVEq
>,
detail::TSVHash,
detail::TSVEq
>;
//
/* /*
Ресурс имеет бинарную часть, из который вырезаны все зависимости. Ресурс имеет бинарную часть, из который вырезаны все зависимости.
Вторая часть это заголовок, которые всегда динамично передаётся с сервера. Вторая часть это заголовок, которые всегда динамично передаётся с сервера.
@@ -100,63 +69,13 @@ public:
std::string Key; std::string Key;
fs::file_time_type Timestamp; fs::file_time_type Timestamp;
// Обезличенный ресурс // Обезличенный ресурс
std::shared_ptr<std::u8string> Resource; std::u8string Resource;
// Его хеш // Его хеш
ResourceFile::Hash_t Hash; ResourceFile::Hash_t Hash;
// Заголовок // Заголовок
std::u8string Header; std::u8string Header;
}; };
struct BindDomainKeyInfo {
std::string Domain;
std::string Key;
};
struct BindHashHeaderInfo {
ResourceId Id;
Hash_t Hash;
std::u8string Header;
};
struct Out_reloadResources {
std::unordered_map<std::string, std::vector<PendingResource>> NewOrChange[(int) AssetType::MAX_ENUM];
std::unordered_map<std::string, std::vector<std::string>> Lost[(int) AssetType::MAX_ENUM];
};
struct Out_applyResourceChange {
std::array<
std::vector<AssetsPreloader::BindHashHeaderInfo>,
static_cast<size_t>(AssetType::MAX_ENUM)
> NewOrChange;
std::array<
std::vector<ResourceId>,
static_cast<size_t>(AssetType::MAX_ENUM)
> Lost;
};
struct Out_bakeId {
// Новые привязки
std::array<
std::vector<BindDomainKeyInfo>,
static_cast<size_t>(AssetType::MAX_ENUM)
> IdToDK;
};
struct Out_fullSync {
std::array<
std::vector<BindDomainKeyInfo>,
static_cast<size_t>(AssetType::MAX_ENUM)
> IdToDK;
std::array<
std::vector<BindHashHeaderInfo>,
static_cast<size_t>(AssetType::MAX_ENUM)
> HashHeaders;
std::vector<std::tuple<AssetType, ResourceId, const MediaResource*>> Resources;
};
struct ReloadStatus { struct ReloadStatus {
/// TODO: callback'и для обновления статусов /// TODO: callback'и для обновления статусов
/// TODO: многоуровневый статус std::vector<std::string>. Этапы/Шаги/Объекты /// TODO: многоуровневый статус std::vector<std::string>. Этапы/Шаги/Объекты
@@ -202,59 +121,125 @@ public:
! Бронирует идентификаторы используя getId(); ! Бронирует идентификаторы используя getId();
instances -> пути к директории с assets или архивы с assets внутри. От низшего приоритета к высшему. instances -> пути к директории с assets или архивы с assets внутри. От низшего приоритета к высшему.
idResolver -> функция получения идентификатора по Тип+Домен+Ключ
onNewResourceParsed -> Callback на обработку распаршенных ресурсов без заголовков
(на стороне сервера хранится в другой сущности, на стороне клиента игнорируется).
status -> обратный отклик о процессе обновления ресурсов. status -> обратный отклик о процессе обновления ресурсов.
ReloadStatus <- новые и потерянные ресурсы. ReloadStatus <- новые и потерянные ресурсы.
*/ */
Out_reloadResources reloadResources(const AssetsRegister& instances, ReloadStatus* status = nullptr); struct Out_checkAndPrepareResourcesUpdate {
// Новые связки Id -> Hash + Header + Timestamp + Path (ресурс новый или изменён)
std::array<
std::vector<
std::tuple<
ResourceId, // Ресурс
ResourceFile::Hash_t, // Хэш ресурса на диске
ResourceHeader, // Хедер ресурса (со всеми зависимостями)
fs::file_time_type, // Время изменения ресурса на диске
fs::path // Путь до ресурса
>
>,
static_cast<size_t>(AssetType::MAX_ENUM)
> ResourceUpdates;
// Используется чтобы эффективно увеличить размер таблиц
std::array<
ResourceId,
static_cast<size_t>(AssetType::MAX_ENUM)
> MaxNewSize;
// Потерянные связки Id (ресурс физически потерян)
std::array<
std::vector<ResourceId>,
static_cast<size_t>(AssetType::MAX_ENUM)
> LostLinks;
/*
Новые пути предоставляющие хеш
(по каким путям можно получить ресурс определённого хеша).
*/
std::unordered_map<
ResourceFile::Hash_t,
std::vector<fs::path>
> HashToPathNew;
/*
Потерянные пути, предоставлявшые ресурсы с данным хешем
(пути по которым уже нельзя получить заданных хеш).
*/
std::unordered_map<
ResourceFile::Hash_t,
std::vector<fs::path>
> HashToPathLost;
};
Out_checkAndPrepareResourcesUpdate checkAndPrepareResourcesUpdate(
const AssetsRegister& instances,
const std::function<ResourceId(EnumAssets type, std::string_view domain, std::string_view key)>& idResolver,
const std::function<void(std::u8string&& resource, ResourceFile::Hash_t hash, fs::path resPath)>& onNewResourceParsed = nullptr,
ReloadStatus* status = nullptr
);
/* /*
Применяет расчитанные изменения. Применяет расчитанные изменения.
Out_applyResourceChange <- Нужно отправить клиентам новые привязки ресурсов Out_applyResourceUpdate <- Нужно отправить клиентам новые привязки ресурсов
id -> hash+header id -> hash+header
*/ */
Out_applyResourceChange applyResourceChange(const Out_reloadResources& orr); struct BindHashHeaderInfo {
ResourceId Id;
ResourceFile::Hash_t Hash;
ResourceHeader Header;
};
/* struct Out_applyResourcesUpdate {
Выдаёт идентификатор ресурса. std::array<
Многопоточно. std::vector<BindHashHeaderInfo>,
Иногда нужно вызывать bakeIdTables чтобы оптимизировать таблицы static_cast<size_t>(EnumAssets::MAX_ENUM)
идентификаторов. При этом никто не должен использовать getId > NewOrUpdates;
*/ };
ResourceId getId(AssetType type, std::string_view domain, std::string_view key);
/* Out_applyResourcesUpdate applyResourcesUpdate(const Out_checkAndPrepareResourcesUpdate& orr);
Оптимизирует таблицы идентификаторов.
Нельзя использовать пока есть вероятность что кто-то использует getId().
Такжке нельзя при выполнении reloadResources().
Out_bakeId <- Нужно отправить подключенным клиентам новые привязки id -> домен+ключ std::array<
*/ std::vector<BindHashHeaderInfo>,
Out_bakeId bakeIdTables(); static_cast<size_t>(EnumAssets::MAX_ENUM)
> collectHashBindings() const
{
std::array<
std::vector<BindHashHeaderInfo>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
> result;
// Выдаёт полный список привязок и ресурсов для новых клиентов. for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); ++type) {
Out_fullSync collectFullSync() const; result[type].reserve(ResourceLinks[type].size());
/* ResourceId counter = 0;
Выдаёт пакет со всеми текущими привязками id -> домен+ключ. for(const auto& [hash, header, _1, _2, _3] : ResourceLinks[type]) {
Используется при подключении новых клиентов. ResourceId id = counter++;
*/ result[type].emplace_back(id, hash, header);
void makeGlobalLinkagePacket() { }
/// TODO: Собрать пакет с IdToDK и сжать его домены и ключи и id -> hash+header }
// Тот же пакет для обновления идентификаторов return result;
std::unreachable();
} }
// Выдаёт ресурс по идентификатору const auto& getResourceLinks() const {
const MediaResource* getResource(AssetType type, uint32_t id) const; return ResourceLinks;
}
// Выдаёт ресурс по хешу struct Out_Resource {
std::optional<std::tuple<AssetType, uint32_t, const MediaResource*>> getResource(const ResourceFile::Hash_t& hash); ResourceFile::Hash_t Hash;
fs::path Path;
};
// Выдаёт зависимости к ресурсам профиля ноды std::optional<Out_Resource> getResource(EnumAssets type, ResourceId id) const {
std::tuple<AssetsNodestate, std::vector<AssetsModel>, std::vector<AssetsTexture>> const auto& rl = ResourceLinks[static_cast<size_t>(type)];
getNodeDependency(const std::string& domain, const std::string& key); if(id >= rl.size() || !rl[id].IsExist)
return std::nullopt;
return Out_Resource{rl[id].Hash, rl[id].Path};
}
private: private:
struct ResourceFindInfo { struct ResourceFindInfo {
@@ -262,6 +247,8 @@ private:
fs::path ArchivePath, Path; fs::path ArchivePath, Path;
// Время изменения файла // Время изменения файла
fs::file_time_type Timestamp; fs::file_time_type Timestamp;
// Идентификатор ресурса
ResourceId Id;
}; };
struct HashHasher { struct HashHasher {
@@ -275,136 +262,32 @@ private:
} }
}; };
#ifndef NDEBUG
// Текущее состояние reloadResources // Текущее состояние reloadResources
std::atomic<bool> _Reloading = false; std::atomic<bool> _Reloading = false;
// Если идентификатор не найден в асинхронной таблице, переходим к работе с синхронной
ResourceId _getIdNew(AssetType type, std::string_view domain, std::string_view key);
Out_reloadResources _reloadResources(const AssetsRegister& instances, ReloadStatus& status);
#ifndef NDEBUG
// Для контроля за режимом слияния ключей
bool DKToIdInBakingMode = false;
#endif #endif
/* struct ResourceLink {
Многопоточная таблица идентификаторов. Новые идентификаторы выделяются в NewDKToId, ResourceFile::Hash_t Hash; // Хэш ресурса на диске
и далее вливаются в основную таблицу при вызове bakeIdTables() /// TODO: клиенту не нужны хедеры
*/ ResourceHeader Header; // Хедер ресурса (со всеми зависимостями)
std::array<IdTable, static_cast<size_t>(AssetType::MAX_ENUM)> DKToId; fs::file_time_type LastWrite; // Время изменения ресурса на диске
/* fs::path Path; // Путь до ресурса
Многопоточная таблица обратного резолва. bool IsExist;
Идентификатор -> домен+ключ };
*/
std::array<std::vector<BindDomainKeyInfo>, static_cast<size_t>(AssetType::MAX_ENUM)> IdToDK;
/* Out_checkAndPrepareResourcesUpdate _checkAndPrepareResourcesUpdate(
Таблица в которой выделяются новые идентификаторы, которых не нашлось в DKToId. const AssetsRegister& instances,
Данный объект одновременно может работать только с одним потоком. const std::function<ResourceId(EnumAssets type, std::string_view domain, std::string_view key)>& idResolver,
*/ const std::function<void(std::u8string&& resource, ResourceFile::Hash_t hash, fs::path resPath)>& onNewResourceParsed,
std::array<TOS::SpinlockObject<IdTable>, static_cast<size_t>(AssetType::MAX_ENUM)> NewDKToId; ReloadStatus& status
/* );
Конец поля идентификаторов, известный клиентам.
Если NextId продвинулся дальше, нужно уведомить клиентов о новых привязках.
*/
std::array<ResourceId, static_cast<size_t>(AssetType::MAX_ENUM)> LastSendId;
/*
Списки в которых пишутся новые привязки. Начала спиской исходят из LastSendId.
Id + LastSendId -> домен+ключ
*/
std::array<TOS::SpinlockObject<std::vector<BindDomainKeyInfo>>, static_cast<size_t>(AssetType::MAX_ENUM)> NewIdToDK;
// Загруженные ресурсы // Привязка Id -> Hash + Header + Timestamp + Path
std::array<std::unordered_map<ResourceId, MediaResource>, static_cast<size_t>(AssetType::MAX_ENUM)> MediaResources; std::array<
// Hash -> ресурс std::vector<ResourceLink>,
std::unordered_map<ResourceFile::Hash_t, std::pair<AssetType, ResourceId>, HashHasher> HashToId; static_cast<size_t>(AssetType::MAX_ENUM)
// Для последовательного выделения идентификаторов > ResourceLinks;
std::array<ResourceId, static_cast<size_t>(AssetType::MAX_ENUM)> NextId;
}; };
inline ResourceId AssetsPreloader::getId(AssetType type, std::string_view domain, std::string_view key) {
#ifndef NDEBUG
assert(!DKToIdInBakingMode);
#endif
const auto& typeTable = DKToId[static_cast<size_t>(type)];
auto domainTable = typeTable.find(domain);
#ifndef NDEBUG
assert(!DKToIdInBakingMode);
#endif
if(domainTable == typeTable.end())
return _getIdNew(type, domain, key);
auto keyTable = domainTable->second.find(key);
if (keyTable == domainTable->second.end())
return _getIdNew(type, domain, key);
return keyTable->second;
return 0;
}
inline ResourceId AssetsPreloader::_getIdNew(AssetType type, std::string_view domain, std::string_view key) {
auto lock = NewDKToId[static_cast<size_t>(type)].lock();
auto iterDomainNewTable = lock->find(domain);
if(iterDomainNewTable == lock->end()) {
iterDomainNewTable = lock->emplace_hint(
iterDomainNewTable,
(std::string) domain,
std::unordered_map<std::string, uint32_t, detail::TSVHash, detail::TSVEq>{}
);
}
auto& domainNewTable = iterDomainNewTable->second;
if(auto iter = domainNewTable.find(key); iter != domainNewTable.end())
return iter->second;
uint32_t id = domainNewTable[(std::string) key] = NextId[static_cast<size_t>(type)]++;
auto lock2 = NewIdToDK[static_cast<size_t>(type)].lock();
lock.unlock();
lock2->emplace_back((std::string) domain, (std::string) key);
return id;
}
inline const AssetsPreloader::MediaResource* AssetsPreloader::getResource(AssetType type, uint32_t id) const {
auto& iterType = MediaResources[static_cast<size_t>(type)];
auto iterRes = iterType.find(id);
if(iterRes == iterType.end())
return nullptr;
return &iterRes->second;
}
inline std::optional<std::tuple<AssetType, uint32_t, const AssetsPreloader::MediaResource*>>
AssetsPreloader::getResource(const ResourceFile::Hash_t& hash)
{
auto iter = HashToId.find(hash);
if(iter == HashToId.end())
return std::nullopt;
auto [type, id] = iter->second;
const MediaResource* res = getResource(type, id);
if(!res) {
HashToId.erase(iter);
return std::nullopt;
}
if(res->Hash != hash) {
HashToId.erase(iter);
return std::nullopt;
}
return std::tuple<AssetType, uint32_t, const MediaResource*>{type, id, res};
}
} }

272
Src/Common/IdProvider.hpp Normal file
View File

@@ -0,0 +1,272 @@
#pragma once
#include "Common/Abstract.hpp"
#include <ankerl/unordered_dense.h>
#include <array>
#include <atomic>
#include <cassert>
#include <optional>
#include <shared_mutex>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include <algorithm>
namespace LV {
template<class Enum = EnumAssets, size_t ShardCount = 64>
class IdProvider {
public:
static constexpr size_t MAX_ENUM = static_cast<size_t>(Enum::MAX_ENUM);
struct BindDomainKeyInfo {
std::string Domain, Key;
};
struct BindDomainKeyViewInfo {
std::string_view Domain, Key;
};
struct KeyHash {
using is_transparent = void;
static inline std::size_t h(std::string_view sv) noexcept {
return std::hash<std::string_view>{}(sv);
}
static inline std::size_t mix(std::size_t a, std::size_t b) noexcept {
a ^= b + 0x9e3779b97f4a7c15ULL + (a << 6) + (a >> 2);
return a;
}
std::size_t operator()(const BindDomainKeyInfo& k) const noexcept {
return mix(h(k.Domain), h(k.Key));
}
std::size_t operator()(const BindDomainKeyViewInfo& kv) const noexcept {
return mix(h(kv.Domain), h(kv.Key));
}
};
struct KeyEq {
using is_transparent = void;
bool operator()(const BindDomainKeyInfo& a, const BindDomainKeyInfo& b) const noexcept {
return a.Domain == b.Domain && a.Key == b.Key;
}
bool operator()(const BindDomainKeyInfo& a, const BindDomainKeyViewInfo& b) const noexcept {
return a.Domain == b.Domain && a.Key == b.Key;
}
bool operator()(const BindDomainKeyViewInfo& a, const BindDomainKeyInfo& b) const noexcept {
return a.Domain == b.Domain && a.Key == b.Key;
}
};
public:
explicit IdProvider() {
for(size_t type = 0; type < MAX_ENUM; ++type) {
_NextId[type].store(1, std::memory_order_relaxed);
_Reverse[type].reserve(1024);
IdToDK[type].push_back({"core", "none"});
auto& sh = _shardFor(static_cast<Enum>(type), "core", "none");
std::unique_lock lk(sh.mutex);
sh.map.emplace(BindDomainKeyInfo{"core", "none"}, 0);
// ensure id 0 has a reverse mapping too
_storeReverse(static_cast<Enum>(type), 0, std::string("core"), std::string("none"));
}
}
/*
Находит или выдаёт идентификатор на запрошенный ресурс.
Функция не требует внешней синхронизации.
*/
inline ResourceId getId(Enum type, std::string_view domain, std::string_view key) {
#ifndef NDEBUG
assert(!DKToIdInBakingMode);
#endif
auto& sh = _shardFor(type, domain, key);
// 1) Поиск в режиме для чтения
{
std::shared_lock lk(sh.mutex);
if(auto it = sh.map.find(BindDomainKeyViewInfo{domain, key}); it != sh.map.end()) {
return it->second;
}
}
// 2) Блокируем и повторно ищем запись (может кто уже успел её добавить)
std::unique_lock lk(sh.mutex);
if (auto it = sh.map.find(BindDomainKeyViewInfo{domain, key}); it != sh.map.end()) {
return it->second;
}
// Выделяем идентификатор
ResourceId id = _NextId[static_cast<size_t>(type)].fetch_add(1, std::memory_order_relaxed);
std::string d(domain);
std::string k(key);
sh.map.emplace(BindDomainKeyInfo{d, k}, id);
sh.newlyInserted.push_back(id);
_storeReverse(type, id, std::move(d), std::move(k));
return id;
}
/*
Переносит все новые идентификаторы в основную таблицу.
В этой реализации "основная таблица" уже основная (forward map обновляется сразу),
а bake() собирает только новые привязки (domain,key) по логам вставок и дополняет IdToDK.
Нельзя использовать пока есть вероятность что кто-то использует getId(), если ты хочешь
строгий debug-контроль как раньше. В релизе это не требуется: bake читает только reverse,
а forward не трогает.
*/
std::array<std::vector<BindDomainKeyInfo>, MAX_ENUM> bake() {
#ifndef NDEBUG
assert(!DKToIdInBakingMode);
DKToIdInBakingMode = true;
struct _tempStruct {
IdProvider* handler;
~_tempStruct() { handler->DKToIdInBakingMode = false; }
} _lock{this};
#endif
std::array<std::vector<BindDomainKeyInfo>, MAX_ENUM> result;
for(size_t t = 0; t < MAX_ENUM; ++t) {
auto type = static_cast<Enum>(t);
// 1) собрать новые id из всех шардов
std::vector<ResourceId> new_ids;
_drainNew(type, new_ids);
if(new_ids.empty())
continue;
// 2) превратить id -> (domain,key) через reverse и вернуть наружу
// + дописать в IdToDK[type] в порядке id (по желанию)
std::sort(new_ids.begin(), new_ids.end());
new_ids.erase(std::unique(new_ids.begin(), new_ids.end()), new_ids.end());
result[t].reserve(new_ids.size());
// reverse читаем под shared lock
std::shared_lock rlk(_ReverseMutex[t]);
for(ResourceId id : new_ids) {
const std::size_t idx = static_cast<std::size_t>(id);
if(idx >= _Reverse[t].size()) {
// теоретически не должно случаться (мы пишем reverse до push в log)
continue;
}
const auto& e = _Reverse[t][idx];
result[t].push_back({e.Domain, e.Key});
}
rlk.unlock();
// 3) дописать в IdToDK (для новых клиентов)
IdToDK[t].append_range(result[t]);
}
return result;
}
// id to DK
std::optional<BindDomainKeyInfo> getDK(Enum type, ResourceId id) {
auto& vec = _Reverse[static_cast<size_t>(type)];
auto& mtx = _ReverseMutex[static_cast<size_t>(type)];
std::unique_lock lk(mtx);
if(id >= vec.size())
return std::nullopt;
return vec[id];
}
// Для отправки новым подключенным клиентам
const std::array<std::vector<BindDomainKeyInfo>, MAX_ENUM>& idToDK() const {
return IdToDK;
}
private:
using Map = ankerl::unordered_dense::map<BindDomainKeyInfo, ResourceId, KeyHash, KeyEq>;
struct Shard {
mutable std::shared_mutex mutex;
Map map;
std::vector<ResourceId> newlyInserted;
};
private:
// Кластер таблиц идентификаторов
std::array<
std::array<Shard, ShardCount>, MAX_ENUM
> _Shards;
// Счётчики идентификаторов
std::array<std::atomic<ResourceId>, MAX_ENUM> _NextId;
// Таблица обратных связок (Id to DK)
std::array<std::vector<BindDomainKeyInfo>, MAX_ENUM> _Reverse;
mutable std::array<std::shared_mutex, MAX_ENUM> _ReverseMutex;
#ifndef NDEBUG
bool DKToIdInBakingMode = false;
#endif
// stable "full sync" table for new clients:
std::array<std::vector<BindDomainKeyInfo>, MAX_ENUM> IdToDK;
private:
Shard& _shardFor(Enum type, const std::string_view domain, const std::string_view key) {
const std::size_t idx = KeyHash{}(BindDomainKeyViewInfo{domain, key}) % ShardCount;
return _Shards[static_cast<size_t>(type)][idx];
}
const Shard& _shardFor(Enum type, const std::string_view domain, const std::string_view key) const {
const std::size_t idx = KeyHash{}(BindDomainKeyViewInfo{domain, key}) % ShardCount;
return _Shards[static_cast<size_t>(type)][idx];
}
void _storeReverse(Enum type, ResourceId id, std::string&& domain, std::string&& key) {
auto& vec = _Reverse[static_cast<size_t>(type)];
auto& mtx = _ReverseMutex[static_cast<size_t>(type)];
const std::size_t idx = static_cast<std::size_t>(id);
std::unique_lock lk(mtx);
if(idx >= vec.size())
vec.resize(idx + 1);
vec[idx] = BindDomainKeyInfo{std::move(domain), std::move(key)};
}
void _drainNew(Enum type, std::vector<ResourceId>& out) {
out.clear();
auto& shards = _Shards[static_cast<size_t>(type)];
// Можно добавить reserve по эвристике
for (auto& sh : shards) {
std::unique_lock lk(sh.mutex);
if (sh.newlyInserted.empty()) continue;
const auto old = out.size();
out.resize(old + sh.newlyInserted.size());
std::copy(sh.newlyInserted.begin(), sh.newlyInserted.end(), out.begin() + old);
sh.newlyInserted.clear();
}
}
};
} // namespace LV

483
Src/Common/Net2.cpp Normal file
View File

@@ -0,0 +1,483 @@
#include "Net2.hpp"
#include <boost/asio/buffer.hpp>
#include <boost/asio/error.hpp>
#include <boost/asio/read.hpp>
#include <boost/asio/write.hpp>
#include <boost/system/system_error.hpp>
#include <algorithm>
#include <tuple>
namespace LV::Net2 {
using namespace TOS;
namespace {
struct HeaderFields {
uint32_t size = 0;
uint16_t type = 0;
Priority priority = Priority::Normal;
FrameFlags flags = FrameFlags::None;
uint32_t streamId = 0;
};
std::array<std::byte, AsyncSocket::kHeaderSize> encodeHeader(const HeaderFields &h) {
std::array<std::byte, AsyncSocket::kHeaderSize> out{};
uint32_t sizeNet = detail::toNetwork(h.size);
uint16_t typeNet = detail::toNetwork(h.type);
uint32_t streamNet = detail::toNetwork(h.streamId);
std::memcpy(out.data(), &sizeNet, sizeof(sizeNet));
std::memcpy(out.data() + 4, &typeNet, sizeof(typeNet));
out[6] = std::byte(static_cast<uint8_t>(h.priority));
out[7] = std::byte(static_cast<uint8_t>(h.flags));
std::memcpy(out.data() + 8, &streamNet, sizeof(streamNet));
return out;
}
HeaderFields decodeHeader(const std::array<std::byte, AsyncSocket::kHeaderSize> &in) {
HeaderFields h{};
std::memcpy(&h.size, in.data(), sizeof(h.size));
std::memcpy(&h.type, in.data() + 4, sizeof(h.type));
h.priority = static_cast<Priority>(std::to_integer<uint8_t>(in[6]));
h.flags = static_cast<FrameFlags>(std::to_integer<uint8_t>(in[7]));
std::memcpy(&h.streamId, in.data() + 8, sizeof(h.streamId));
h.size = detail::fromNetwork(h.size);
h.type = detail::fromNetwork(h.type);
h.streamId = detail::fromNetwork(h.streamId);
return h;
}
} // namespace
PacketWriter& PacketWriter::writeBytes(std::span<const std::byte> data) {
Buffer.insert(Buffer.end(), data.begin(), data.end());
return *this;
}
PacketWriter& PacketWriter::writeString(std::string_view str) {
write<uint32_t>(static_cast<uint32_t>(str.size()));
auto bytes = std::as_bytes(std::span<const char>(str.data(), str.size()));
Buffer.insert(Buffer.end(), bytes.begin(), bytes.end());
return *this;
}
std::vector<std::byte> PacketWriter::release() {
std::vector<std::byte> out = std::move(Buffer);
Buffer.clear();
return out;
}
void PacketWriter::clear() {
Buffer.clear();
}
PacketReader::PacketReader(std::span<const std::byte> data)
: Data(data)
{
}
void PacketReader::readBytes(std::span<std::byte> out) {
require(out.size());
std::memcpy(out.data(), Data.data() + Pos, out.size());
Pos += out.size();
}
std::string PacketReader::readString() {
uint32_t size = read<uint32_t>();
require(size);
std::string out(size, '\0');
std::memcpy(out.data(), Data.data() + Pos, size);
Pos += size;
return out;
}
void PacketReader::require(size_t size) {
if(Data.size() - Pos < size)
MAKE_ERROR("Net2::PacketReader: not enough data");
}
SocketServer::SocketServer(asio::io_context &ioc, std::function<coro<>(tcp::socket)> &&onConnect, uint16_t port)
: AsyncObject(ioc), Acceptor(ioc, tcp::endpoint(tcp::v4(), port))
{
assert(onConnect);
co_spawn(run(std::move(onConnect)));
}
bool SocketServer::isStopped() const {
return !Acceptor.is_open();
}
uint16_t SocketServer::getPort() const {
return Acceptor.local_endpoint().port();
}
coro<void> SocketServer::run(std::function<coro<>(tcp::socket)> onConnect) {
while(true) {
try {
co_spawn(onConnect(co_await Acceptor.async_accept()));
} catch(const std::exception &exc) {
if(const boost::system::system_error *errc = dynamic_cast<const boost::system::system_error*>(&exc);
errc && (errc->code() == asio::error::operation_aborted || errc->code() == asio::error::bad_descriptor))
break;
}
}
}
AsyncSocket::SendQueue::SendQueue(asio::io_context &ioc)
: semaphore(ioc)
{
semaphore.expires_at(std::chrono::steady_clock::time_point::max());
}
bool AsyncSocket::SendQueue::empty() const {
for(const auto &queue : queues) {
if(!queue.empty())
return false;
}
return true;
}
AsyncSocket::AsyncSocket(asio::io_context &ioc, tcp::socket &&socket, Limits limits)
: AsyncObject(ioc), LimitsCfg(limits), Socket(std::move(socket)), Outgoing(ioc)
{
Context = std::make_shared<AsyncContext>();
boost::asio::socket_base::linger optionLinger(true, 4);
Socket.set_option(optionLinger);
boost::asio::ip::tcp::no_delay optionNoDelay(true);
Socket.set_option(optionNoDelay);
co_spawn(sendLoop());
}
AsyncSocket::~AsyncSocket() {
if(Context)
Context->needShutdown.store(true);
{
boost::lock_guard lock(Outgoing.mtx);
Outgoing.semaphore.cancel();
WorkDeadline.cancel();
}
if(Socket.is_open())
try { Socket.close(); } catch(...) {}
}
void AsyncSocket::enqueue(OutgoingMessage &&msg) {
if(msg.payload.size() > LimitsCfg.maxMessageSize) {
setError("Net2::AsyncSocket: message too large");
close();
return;
}
boost::unique_lock lock(Outgoing.mtx);
const size_t msgSize = msg.payload.size();
const size_t lowIndex = static_cast<size_t>(Priority::Low);
if(msg.priority == Priority::Low) {
while(Outgoing.bytesInLow + msgSize > LimitsCfg.maxLowPriorityBytes && !Outgoing.queues[lowIndex].empty()) {
Outgoing.bytesInQueue -= Outgoing.queues[lowIndex].front().payload.size();
Outgoing.bytesInLow -= Outgoing.queues[lowIndex].front().payload.size();
Outgoing.queues[lowIndex].pop_front();
}
if(Outgoing.bytesInLow + msgSize > LimitsCfg.maxLowPriorityBytes) {
return;
}
}
if(Outgoing.bytesInQueue + msgSize > LimitsCfg.maxQueueBytes) {
dropLow(msgSize);
if(Outgoing.bytesInQueue + msgSize > LimitsCfg.maxQueueBytes) {
if(msg.dropIfOverloaded)
return;
setError("Net2::AsyncSocket: send queue overflow");
close();
return;
}
}
const size_t idx = static_cast<size_t>(msg.priority);
Outgoing.bytesInQueue += msgSize;
if(msg.priority == Priority::Low)
Outgoing.bytesInLow += msgSize;
Outgoing.queues[idx].push_back(std::move(msg));
if(Outgoing.waiting) {
Outgoing.waiting = false;
Outgoing.semaphore.cancel();
Outgoing.semaphore.expires_at(std::chrono::steady_clock::time_point::max());
}
}
coro<IncomingMessage> AsyncSocket::readMessage() {
while(true) {
std::array<std::byte, kHeaderSize> headerBytes{};
co_await readExact(headerBytes.data(), headerBytes.size());
HeaderFields header = decodeHeader(headerBytes);
if(header.size > LimitsCfg.maxFrameSize)
MAKE_ERROR("Net2::AsyncSocket: frame too large");
std::vector<std::byte> chunk(header.size);
if(header.size)
co_await readExact(chunk.data(), chunk.size());
if(header.streamId != 0) {
if(Fragments.size() >= LimitsCfg.maxOpenStreams && !Fragments.contains(header.streamId))
MAKE_ERROR("Net2::AsyncSocket: too many open streams");
FragmentState &state = Fragments[header.streamId];
if(state.data.empty()) {
state.type = header.type;
state.priority = header.priority;
}
if(state.data.size() + chunk.size() > LimitsCfg.maxMessageSize)
MAKE_ERROR("Net2::AsyncSocket: reassembled message too large");
state.data.insert(state.data.end(), chunk.begin(), chunk.end());
if(!hasFlag(header.flags, FrameFlags::HasMore)) {
IncomingMessage msg{state.type, state.priority, std::move(state.data)};
Fragments.erase(header.streamId);
co_return msg;
}
continue;
}
if(hasFlag(header.flags, FrameFlags::HasMore))
MAKE_ERROR("Net2::AsyncSocket: stream id missing for fragmented frame");
IncomingMessage msg{header.type, header.priority, std::move(chunk)};
co_return msg;
}
}
coro<> AsyncSocket::readLoop(std::function<coro<>(IncomingMessage&&)> onMessage) {
while(isAlive()) {
IncomingMessage msg = co_await readMessage();
co_await onMessage(std::move(msg));
}
}
void AsyncSocket::closeRead() {
if(Socket.is_open() && !Context->readClosed.exchange(true)) {
try { Socket.shutdown(boost::asio::socket_base::shutdown_receive); } catch(...) {}
}
}
void AsyncSocket::close() {
if(Context)
Context->needShutdown.store(true);
if(Socket.is_open())
try { Socket.close(); } catch(...) {}
}
bool AsyncSocket::isAlive() const {
return Context && !Context->needShutdown.load() && !Context->senderStopped.load() && Socket.is_open();
}
std::string AsyncSocket::getError() const {
boost::lock_guard lock(Context->errorMtx);
return Context->error;
}
coro<> AsyncSocket::sendLoop() {
try {
while(!Context->needShutdown.load()) {
OutgoingMessage msg;
{
boost::unique_lock lock(Outgoing.mtx);
if(Outgoing.empty()) {
Outgoing.waiting = true;
auto coroutine = Outgoing.semaphore.async_wait();
lock.unlock();
try { co_await std::move(coroutine); } catch(...) {}
continue;
}
if(!popNext(msg))
continue;
}
co_await sendMessage(std::move(msg));
}
} catch(const std::exception &exc) {
setError(exc.what());
} catch(...) {
setError("Net2::AsyncSocket: send loop stopped");
}
Context->senderStopped.store(true);
}
coro<> AsyncSocket::sendMessage(OutgoingMessage &&msg) {
const size_t total = msg.payload.size();
if(total <= LimitsCfg.maxFrameSize) {
co_await sendFrame(msg.type, msg.priority, FrameFlags::None, 0, msg.payload);
co_return;
}
if(!msg.allowFragment) {
setError("Net2::AsyncSocket: message requires fragmentation");
close();
co_return;
}
uint32_t streamId = NextStreamId++;
if(streamId == 0)
streamId = NextStreamId++;
size_t offset = 0;
while(offset < total) {
const size_t chunk = std::min(LimitsCfg.maxFrameSize, total - offset);
const bool more = (offset + chunk) < total;
FrameFlags flags = more ? FrameFlags::HasMore : FrameFlags::None;
std::span<const std::byte> view(msg.payload.data() + offset, chunk);
co_await sendFrame(msg.type, msg.priority, flags, streamId, view);
offset += chunk;
}
}
coro<> AsyncSocket::sendFrame(uint16_t type, Priority priority, FrameFlags flags, uint32_t streamId,
std::span<const std::byte> payload) {
HeaderFields header{
.size = static_cast<uint32_t>(payload.size()),
.type = type,
.priority = priority,
.flags = flags,
.streamId = streamId
};
auto headerBytes = encodeHeader(header);
std::array<asio::const_buffer, 2> buffers{
asio::buffer(headerBytes),
asio::buffer(payload.data(), payload.size())
};
if(payload.empty())
co_await asio::async_write(Socket, asio::buffer(headerBytes));
else
co_await asio::async_write(Socket, buffers);
}
coro<> AsyncSocket::readExact(std::byte *data, size_t size) {
if(size == 0)
co_return;
co_await asio::async_read(Socket, asio::buffer(data, size));
}
bool AsyncSocket::popNext(OutgoingMessage &out) {
static constexpr int kWeights[4] = {8, 4, 2, 1};
for(int attempt = 0; attempt < 4; ++attempt) {
const uint8_t idx = static_cast<uint8_t>((Outgoing.nextIndex + attempt) % 4);
auto &queue = Outgoing.queues[idx];
if(queue.empty())
continue;
if(Outgoing.credits[idx] <= 0)
Outgoing.credits[idx] = kWeights[idx];
if(Outgoing.credits[idx] <= 0)
continue;
out = std::move(queue.front());
queue.pop_front();
Outgoing.credits[idx]--;
Outgoing.nextIndex = idx;
const size_t msgSize = out.payload.size();
Outgoing.bytesInQueue -= msgSize;
if(idx == static_cast<uint8_t>(Priority::Low))
Outgoing.bytesInLow -= msgSize;
return true;
}
for(int i = 0; i < 4; ++i)
Outgoing.credits[i] = kWeights[i];
return false;
}
void AsyncSocket::dropLow(size_t needBytes) {
const size_t lowIndex = static_cast<size_t>(Priority::Low);
while(Outgoing.bytesInQueue + needBytes > LimitsCfg.maxQueueBytes && !Outgoing.queues[lowIndex].empty()) {
const size_t size = Outgoing.queues[lowIndex].front().payload.size();
Outgoing.bytesInQueue -= size;
Outgoing.bytesInLow -= size;
Outgoing.queues[lowIndex].pop_front();
}
}
void AsyncSocket::setError(const std::string &msg) {
if(!Context)
return;
boost::lock_guard lock(Context->errorMtx);
Context->error = msg;
}
coro<tcp::socket> asyncConnectTo(const std::string &address,
std::function<void(const std::string&)> onProgress) {
std::string progress;
auto addLog = [&](const std::string &msg) {
progress += '\n';
progress += msg;
if(onProgress)
onProgress('\n' + msg);
};
auto ioc = co_await asio::this_coro::executor;
addLog("Parsing address " + address);
auto re = Str::match(address, "((?:\\[[\\d\\w:]+\\])|(?:[\\d\\.]+))(?:\\:(\\d+))?");
std::vector<std::tuple<tcp::endpoint, std::string>> eps;
if(!re) {
re = Str::match(address, "([-_\\.\\w\\d]+)(?:\\:(\\d+))?");
if(!re)
MAKE_ERROR("Failed to parse address");
tcp::resolver resv{ioc};
tcp::resolver::results_type result;
addLog("Resolving name...");
result = co_await resv.async_resolve(*re->at(1), re->at(2) ? *re->at(2) : "7890");
addLog("Got " + std::to_string(result.size()) + " endpoints");
for(auto iter : result) {
std::string addr = iter.endpoint().address().to_string() + ':' + std::to_string(iter.endpoint().port());
std::string hostname = iter.host_name();
if(hostname == addr)
addLog("ep: " + addr);
else
addLog("ep: " + hostname + " (" + addr + ')');
eps.emplace_back(iter.endpoint(), iter.host_name());
}
} else {
eps.emplace_back(tcp::endpoint{asio::ip::make_address(*re->at(1)),
static_cast<uint16_t>(re->at(2) ? Str::toVal<int>(*re->at(2)) : 7890)},
*re->at(1));
}
for(auto [ep, hostname] : eps) {
addLog("Connecting to " + hostname + " (" + ep.address().to_string() + ':'
+ std::to_string(ep.port()) + ")");
try {
tcp::socket sock{ioc};
co_await sock.async_connect(ep);
addLog("Connected");
co_return sock;
} catch(const std::exception &exc) {
addLog(std::string("Connect failed: ") + exc.what());
}
}
MAKE_ERROR("Unable to connect to server");
}
} // namespace LV::Net2

227
Src/Common/Net2.hpp Normal file
View File

@@ -0,0 +1,227 @@
#pragma once
#include "Async.hpp"
#include "TOSLib.hpp"
#include <boost/asio.hpp>
#include <boost/thread.hpp>
#include <array>
#include <bit>
#include <atomic>
#include <cassert>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <deque>
#include <memory>
#include <optional>
#include <span>
#include <string>
#include <string_view>
#include <type_traits>
#include <unordered_map>
#include <vector>
namespace LV::Net2 {
namespace detail {
constexpr bool kLittleEndian = (std::endian::native == std::endian::little);
template<typename T>
requires std::is_integral_v<T>
inline T toNetwork(T value) {
if constexpr (kLittleEndian && sizeof(T) > 1)
return std::byteswap(value);
return value;
}
template<typename T>
requires std::is_floating_point_v<T>
inline T toNetwork(T value) {
using U = std::conditional_t<sizeof(T) == 4, uint32_t, uint64_t>;
U u = std::bit_cast<U>(value);
u = toNetwork(u);
return std::bit_cast<T>(u);
}
template<typename T>
inline T fromNetwork(T value) {
return toNetwork(value);
}
} // namespace detail
enum class Priority : uint8_t {
Realtime = 0,
High = 1,
Normal = 2,
Low = 3
};
enum class FrameFlags : uint8_t {
None = 0,
HasMore = 1
};
inline FrameFlags operator|(FrameFlags a, FrameFlags b) {
return static_cast<FrameFlags>(static_cast<uint8_t>(a) | static_cast<uint8_t>(b));
}
inline bool hasFlag(FrameFlags value, FrameFlags flag) {
return (static_cast<uint8_t>(value) & static_cast<uint8_t>(flag)) != 0;
}
struct Limits {
size_t maxFrameSize = 1 << 24;
size_t maxMessageSize = 1 << 26;
size_t maxQueueBytes = 1 << 27;
size_t maxLowPriorityBytes = 1 << 26;
size_t maxOpenStreams = 64;
};
struct OutgoingMessage {
uint16_t type = 0;
Priority priority = Priority::Normal;
bool dropIfOverloaded = false;
bool allowFragment = true;
std::vector<std::byte> payload;
};
struct IncomingMessage {
uint16_t type = 0;
Priority priority = Priority::Normal;
std::vector<std::byte> payload;
};
class PacketWriter {
public:
PacketWriter& writeBytes(std::span<const std::byte> data);
template<typename T>
requires (std::is_integral_v<T> || std::is_floating_point_v<T>)
PacketWriter& write(T value) {
T net = detail::toNetwork(value);
std::array<std::byte, sizeof(T)> bytes{};
std::memcpy(bytes.data(), &net, sizeof(T));
Buffer.insert(Buffer.end(), bytes.begin(), bytes.end());
return *this;
}
PacketWriter& writeString(std::string_view str);
const std::vector<std::byte>& data() const { return Buffer; }
std::vector<std::byte> release();
void clear();
private:
std::vector<std::byte> Buffer;
};
class PacketReader {
public:
explicit PacketReader(std::span<const std::byte> data);
template<typename T>
requires (std::is_integral_v<T> || std::is_floating_point_v<T>)
T read() {
require(sizeof(T));
T net{};
std::memcpy(&net, Data.data() + Pos, sizeof(T));
Pos += sizeof(T);
return detail::fromNetwork(net);
}
void readBytes(std::span<std::byte> out);
std::string readString();
bool empty() const { return Pos >= Data.size(); }
size_t remaining() const { return Data.size() - Pos; }
private:
void require(size_t size);
size_t Pos = 0;
std::span<const std::byte> Data;
};
class SocketServer : public AsyncObject {
public:
SocketServer(asio::io_context &ioc, std::function<coro<>(tcp::socket)> &&onConnect, uint16_t port = 0);
bool isStopped() const;
uint16_t getPort() const;
private:
coro<void> run(std::function<coro<>(tcp::socket)> onConnect);
tcp::acceptor Acceptor;
};
class AsyncSocket : public AsyncObject {
public:
static constexpr size_t kHeaderSize = 12;
AsyncSocket(asio::io_context &ioc, tcp::socket &&socket, Limits limits = {});
~AsyncSocket();
void enqueue(OutgoingMessage &&msg);
coro<IncomingMessage> readMessage();
coro<> readLoop(std::function<coro<>(IncomingMessage&&)> onMessage);
void closeRead();
void close();
bool isAlive() const;
std::string getError() const;
private:
struct FragmentState {
uint16_t type = 0;
Priority priority = Priority::Normal;
std::vector<std::byte> data;
};
struct AsyncContext {
std::atomic_bool needShutdown{false};
std::atomic_bool senderStopped{false};
std::atomic_bool readClosed{false};
boost::mutex errorMtx;
std::string error;
};
struct SendQueue {
boost::mutex mtx;
bool waiting = false;
asio::steady_timer semaphore;
std::deque<OutgoingMessage> queues[4];
size_t bytesInQueue = 0;
size_t bytesInLow = 0;
uint8_t nextIndex = 0;
int credits[4] = {8, 4, 2, 1};
explicit SendQueue(asio::io_context &ioc);
bool empty() const;
};
coro<> sendLoop();
coro<> sendMessage(OutgoingMessage &&msg);
coro<> sendFrame(uint16_t type, Priority priority, FrameFlags flags, uint32_t streamId,
std::span<const std::byte> payload);
coro<> readExact(std::byte *data, size_t size);
bool popNext(OutgoingMessage &out);
void dropLow(size_t needBytes);
void setError(const std::string &msg);
Limits LimitsCfg;
tcp::socket Socket;
SendQueue Outgoing;
std::shared_ptr<AsyncContext> Context;
std::unordered_map<uint32_t, FragmentState> Fragments;
uint32_t NextStreamId = 1;
};
coro<tcp::socket> asyncConnectTo(const std::string &address,
std::function<void(const std::string&)> onProgress = nullptr);
} // namespace LV::Net2

View File

@@ -92,6 +92,7 @@ enum struct ToClient : uint8_t {
AssetsInitSend, // Начало отправки запрошенного клиентом ресурса AssetsInitSend, // Начало отправки запрошенного клиентом ресурса
AssetsNextSend, // Продолжение отправки ресурса AssetsNextSend, // Продолжение отправки ресурса
DefinitionsFull, // Полная информация о профилях контента
DefinitionsUpdate, // Обновление и потеря профилей контента (воксели, ноды, сущности, миры, ...) DefinitionsUpdate, // Обновление и потеря профилей контента (воксели, ноды, сущности, миры, ...)
ChunkVoxels, // Обновление вокселей чанка ChunkVoxels, // Обновление вокселей чанка

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -34,24 +34,6 @@ using PlayerId_t = ResourceId;
using DefGeneratorId_t = ResourceId; using DefGeneratorId_t = ResourceId;
/*
Сервер загружает информацию о локальных текстурах
Пересмотр списка текстур?
Динамичные текстуры?
*/
struct ResourceFile {
using Hash_t = sha2::sha256_hash; // boost::uuids::detail::sha1::digest_type;
Hash_t Hash;
std::vector<std::byte> Data;
void calcHash() {
Hash = sha2::sha256((const uint8_t*) Data.data(), Data.size());
}
};
struct ServerTime { struct ServerTime {
uint32_t Seconds : 24, Sub : 8; uint32_t Seconds : 24, Sub : 8;
}; };
@@ -236,6 +218,7 @@ public:
} }
DefEntityId getDefId() const { return DefId; } DefEntityId getDefId() const { return DefId; }
void setDefId(DefEntityId defId) { DefId = defId; }
}; };
template<typename Vec> template<typename Vec>
@@ -507,4 +490,4 @@ struct ContentViewCircle {
int16_t Range; int16_t Range;
}; };
} }

View File

@@ -0,0 +1,129 @@
#pragma once
#include "Common/Abstract.hpp"
#include "Common/IdProvider.hpp"
#include "Common/AssetsPreloader.hpp"
#include <unordered_map>
namespace LV::Server {
class AssetsManager : public IdProvider<EnumAssets>, protected AssetsPreloader {
public:
using BindHashHeaderInfo = AssetsManager::BindHashHeaderInfo;
struct Out_checkAndPrepareResourcesUpdate : public AssetsPreloader::Out_checkAndPrepareResourcesUpdate {
Out_checkAndPrepareResourcesUpdate(AssetsPreloader::Out_checkAndPrepareResourcesUpdate&& obj)
: AssetsPreloader::Out_checkAndPrepareResourcesUpdate(std::move(obj))
{}
std::unordered_map<ResourceFile::Hash_t, std::u8string> NewHeadless;
};
Out_checkAndPrepareResourcesUpdate checkAndPrepareResourcesUpdate(
const AssetsRegister& instances,
ReloadStatus* status = nullptr
) {
std::unordered_map<ResourceFile::Hash_t, std::u8string> newHeadless;
Out_checkAndPrepareResourcesUpdate result = AssetsPreloader::checkAndPrepareResourcesUpdate(
instances,
[&](EnumAssets type, std::string_view domain, std::string_view key) { return getId(type, domain, key); },
[&](std::u8string&& resource, ResourceFile::Hash_t hash, fs::path resPath) { newHeadless.emplace(hash, std::move(resource)); },
status
);
result.NewHeadless = std::move(newHeadless);
return result;
}
struct Out_applyResourcesUpdate : public AssetsPreloader::Out_applyResourcesUpdate {
Out_applyResourcesUpdate(AssetsPreloader::Out_applyResourcesUpdate&& obj)
: AssetsPreloader::Out_applyResourcesUpdate(std::move(obj))
{}
};
Out_applyResourcesUpdate applyResourcesUpdate(Out_checkAndPrepareResourcesUpdate& orr) {
Out_applyResourcesUpdate result = AssetsPreloader::applyResourcesUpdate(orr);
{
static TOS::Logger LOG = "Server>AssetsManager";
for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); ++type) {
if(result.NewOrUpdates[type].empty())
continue;
EnumAssets typeEnum = static_cast<EnumAssets>(type);
const char* typeName = ::EnumAssetsToDirectory(typeEnum);
for(const auto& bind : result.NewOrUpdates[type]) {
auto dk = getDK(typeEnum, bind.Id);
if(!dk)
continue;
LOG.debug()
<< typeName << ": "
<< dk->Domain << '+' << dk->Key << " -> " << bind.Id;
}
}
}
for(auto& [hash, data] : orr.NewHeadless) {
Resources.emplace(hash, ResourceHashData{0, std::make_shared<std::u8string>(std::move(data))});
}
for(auto& [hash, pathes] : orr.HashToPathNew) {
auto iter = Resources.find(hash);
assert(iter != Resources.end());
iter->second.RefCount += pathes.size();
}
for(auto& [hash, pathes] : orr.HashToPathLost) {
auto iter = Resources.find(hash);
assert(iter != Resources.end());
iter->second.RefCount -= pathes.size();
if(iter->second.RefCount == 0)
Resources.erase(iter);
}
return result;
}
std::vector<std::tuple<ResourceFile::Hash_t, std::shared_ptr<const std::u8string>>>
getResources(const std::vector<ResourceFile::Hash_t>& hashes) const
{
std::vector<std::tuple<ResourceFile::Hash_t, std::shared_ptr<const std::u8string>>> result;
result.reserve(hashes.size());
for(const auto& hash : hashes) {
auto iter = Resources.find(hash);
if(iter == Resources.end())
continue;
result.emplace_back(hash, iter->second.Data);
}
return result;
}
std::array<
std::vector<BindHashHeaderInfo>,
static_cast<size_t>(EnumAssets::MAX_ENUM)
> collectHashBindings() const {
return AssetsPreloader::collectHashBindings();
}
private:
struct ResourceHashData {
size_t RefCount;
std::shared_ptr<std::u8string> Data;
};
std::unordered_map<
ResourceFile::Hash_t,
ResourceHashData
> Resources;
};
}

View File

@@ -1,25 +1,35 @@
#include "ContentManager.hpp" #include "ContentManager.hpp"
#include "Common/Abstract.hpp" #include "Common/Abstract.hpp"
#include <algorithm> #include <algorithm>
#include <optional>
#include <utility>
namespace LV::Server { namespace LV::Server {
ContentManager::ContentManager(AssetsPreloader& am) ContentManager::ContentManager(AssetsManager& am)
: AM(am) : AM(am)
{ {
std::fill(std::begin(NextId), std::end(NextId), 1);
} }
ContentManager::~ContentManager() = default; ContentManager::~ContentManager() = default;
void ContentManager::registerBase_Node(ResourceId id, const std::string& domain, const std::string& key, const sol::table& profile) { void ContentManager::registerBase_Node(ResourceId id, const std::string& domain, const std::string& key, const sol::table& profile) {
std::optional<DefNode>& node = getEntry_Node(id); std::optional<DefNode_Base>* basePtr;
if(!node)
node.emplace();
DefNode& def = *node; {
def.Domain = domain; size_t entryIndex = id / TableEntry<DefNode_Base>::ChunkSize;
def.Key = key; size_t entryId = id % TableEntry<DefNode_Base>::ChunkSize;
size_t need = entryIndex+1-Profiles_Base_Node.size();
for(size_t iter = 0; iter < need; iter++) {
Profiles_Base_Node.emplace_back(std::make_unique<TableEntry<DefNode_Base>>());
}
basePtr = &Profiles_Base_Node[entryIndex]->Entries[entryId];
*basePtr = DefNode_Base();
}
DefNode_Base& def = **basePtr;
{ {
std::optional<std::variant<std::string, sol::table>> parent = profile.get<std::optional<std::variant<std::string, sol::table>>>("parent"); std::optional<std::variant<std::string, sol::table>> parent = profile.get<std::optional<std::variant<std::string, sol::table>>>("parent");
@@ -93,7 +103,7 @@ void ContentManager::registerBase_Entity(ResourceId id, const std::string& domai
void ContentManager::registerBase(EnumDefContent type, const std::string& domain, const std::string& key, const sol::table& profile) void ContentManager::registerBase(EnumDefContent type, const std::string& domain, const std::string& key, const sol::table& profile)
{ {
ResourceId id = getId(type, domain, key); ResourceId id = getId(type, domain, key);
ProfileChanges[(int) type].push_back(id); ProfileChanges[static_cast<size_t>(type)].push_back(id);
if(type == EnumDefContent::Node) if(type == EnumDefContent::Node)
registerBase_Node(id, domain, key, profile); registerBase_Node(id, domain, key, profile);
@@ -123,56 +133,203 @@ void ContentManager::unRegisterModifier(EnumDefContent type, const std::string&
ProfileChanges[(int) type].push_back(id); ProfileChanges[(int) type].push_back(id);
} }
void ContentManager::markAllProfilesDirty(EnumDefContent type) { // void ContentManager::markAllProfilesDirty(EnumDefContent type) {
const auto &table = ContentKeyToId[(int) type]; // const auto &table = this->idToDK()[(int) type];
for(const auto& domainPair : table) { // size_t counter = 0;
for(const auto& keyPair : domainPair.second) { // for(const auto& [domain, key] : table) {
ProfileChanges[(int) type].push_back(keyPair.second); // ProfileChanges[static_cast<size_t>(type)].push_back(counter++);
} // }
} // }
}
std::vector<ResourceId> ContentManager::collectProfileIds(EnumDefContent type) const { template<class type, class modType>
std::vector<ResourceId> ids; void ContentManager::buildEndProfilesByType(auto& profiles, auto enumType, auto& base, auto& keys, auto& result, auto& modsTable) {
const auto &table = ContentKeyToId[(int) type]; // Расширяем таблицу итоговых профилей до нужного количества
if(!keys.empty()) {
size_t need = keys.back() / TableEntry<type>::ChunkSize;
if(need >= profiles.size()) {
profiles.reserve(need);
for(const auto& domainPair : table) { for(size_t iter = 0; iter <= need-profiles.size(); ++iter)
for(const auto& keyPair : domainPair.second) { profiles.emplace_back(std::make_unique<TableEntry<type>>());
ids.push_back(keyPair.second);
} }
} }
std::sort(ids.begin(), ids.end()); TOS::Logger("CM").debug() << "type: " << static_cast<size_t>(enumType);
auto last = std::unique(ids.begin(), ids.end());
ids.erase(last, ids.end()); // Пересчитываем профили
return ids; for(size_t id : keys) {
size_t entryIndex = id / TableEntry<type>::ChunkSize;
size_t subIndex = id % TableEntry<type>::ChunkSize;
if(
entryIndex >= base.size()
|| !base[entryIndex]->Entries[subIndex]
) {
// Базовый профиль не существует
profiles[entryIndex]->Entries[subIndex] = std::nullopt;
// Уведомляем о потере профиля
result.LostProfiles[static_cast<size_t>(enumType)].push_back(id);
} else {
// Собираем конечный профиль
std::vector<std::tuple<std::string, modType>> mods_default, *mods = &mods_default;
auto iter = modsTable.find(id);
if(iter != modsTable.end())
mods = &iter->second;
std::optional<BindDomainKeyInfo> dk = getDK(enumType, id);
assert(dk);
TOS::Logger("CM").debug() << "\t" << dk->Domain << ":" << dk->Key << " -> " << id;
profiles[entryIndex]->Entries[subIndex] = base[entryIndex]->Entries[subIndex]->compile(AM, *this, dk->Domain, dk->Key, *mods);
}
}
} }
ContentManager::Out_buildEndProfiles ContentManager::buildEndProfiles() { ContentManager::Out_buildEndProfiles ContentManager::buildEndProfiles() {
Out_buildEndProfiles result; Out_buildEndProfiles result;
for(int type = 0; type < (int) EnumDefContent::MAX_ENUM; type++) { for(int type = 0; type < (int) EnumDefContent::MAX_ENUM; type++) {
std::shared_lock lock(Profiles_Mtx[type]);
auto& keys = ProfileChanges[type]; auto& keys = ProfileChanges[type];
std::sort(keys.begin(), keys.end()); std::sort(keys.begin(), keys.end());
auto iterErase = std::unique(keys.begin(), keys.end()); auto iterErase = std::unique(keys.begin(), keys.end());
keys.erase(iterErase, keys.end()); keys.erase(iterErase, keys.end());
}
for(ResourceId id : ProfileChanges[(int) EnumDefContent::Node]) { switch(type) {
std::optional<DefNode>& node = getEntry_Node(id); case 0: buildEndProfilesByType<DefVoxel, DefVoxel_Mod> (Profiles_Voxel, EnumDefContent::Voxel, Profiles_Base_Voxel, keys, result, Profiles_Mod_Voxel); break;
if(!node) { case 1: buildEndProfilesByType<DefNode, DefNode_Mod> (Profiles_Node, EnumDefContent::Node, Profiles_Base_Node, keys, result, Profiles_Mod_Node); break;
continue; case 2: buildEndProfilesByType<DefWorld, DefWorld_Mod> (Profiles_World, EnumDefContent::World, Profiles_Base_World, keys, result, Profiles_Mod_World); break;
case 3: buildEndProfilesByType<DefPortal, DefPortal_Mod> (Profiles_Portal, EnumDefContent::Portal, Profiles_Base_Portal, keys, result, Profiles_Mod_Portal); break;
case 4: buildEndProfilesByType<DefEntity, DefEntity_Mod> (Profiles_Entity, EnumDefContent::Entity, Profiles_Base_Entity, keys, result, Profiles_Mod_Entity); break;
case 5: buildEndProfilesByType<DefItem, DefItem_Mod> (Profiles_Item, EnumDefContent::Item, Profiles_Base_Item, keys, result, Profiles_Mod_Item); break;
default: std::unreachable();
} }
auto [nodestateId, assetsModel, assetsTexture]
= AM.getNodeDependency(node->Domain, node->Key);
node->NodestateId = nodestateId;
node->ModelDeps = std::move(assetsModel);
node->TextureDeps = std::move(assetsTexture);
} }
return result; return result;
} }
ContentManager::Out_getAllProfiles ContentManager::getAllProfiles() {
Out_getAllProfiles result;
size_t counter;
{
std::shared_lock lock(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Voxel)]);
result.ProfilesIds_Voxel.reserve(Profiles_Voxel.size()*TableEntry<DefVoxel>::ChunkSize);
counter = 0;
for(const auto& entry : Profiles_Voxel)
for(const auto& item : entry->Entries) {
size_t id = counter++;
if(item)
result.ProfilesIds_Voxel.push_back(id);
}
result.ProfilesIds_Voxel.shrink_to_fit();
result.Profiles_Voxel.reserve(result.ProfilesIds_Voxel.size());
for(const auto& entry : Profiles_Voxel)
for(const auto& item : entry->Entries)
if(item)
result.Profiles_Voxel.push_back(&item.value());
}
{
std::shared_lock lock(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Node)]);
result.ProfilesIds_Node.reserve(Profiles_Node.size()*TableEntry<DefNode>::ChunkSize);
counter = 0;
for(const auto& entry : Profiles_Node)
for(const auto& item : entry->Entries) {
size_t id = counter++;
if(item)
result.ProfilesIds_Node.push_back(id);
}
result.ProfilesIds_Node.shrink_to_fit();
result.Profiles_Node.reserve(result.ProfilesIds_Node.size());
for(const auto& entry : Profiles_Node)
for(const auto& item : entry->Entries)
if(item)
result.Profiles_Node.push_back(&item.value());
}
{
std::shared_lock lock(Profiles_Mtx[static_cast<size_t>(EnumDefContent::World)]);
result.ProfilesIds_World.reserve(Profiles_World.size()*TableEntry<DefWorld>::ChunkSize);
counter = 0;
for(const auto& entry : Profiles_World)
for(const auto& item : entry->Entries) {
size_t id = counter++;
if(item)
result.ProfilesIds_World.push_back(id);
}
result.ProfilesIds_World.shrink_to_fit();
result.Profiles_World.reserve(result.ProfilesIds_World.size());
for(const auto& entry : Profiles_World)
for(const auto& item : entry->Entries)
if(item)
result.Profiles_World.push_back(&item.value());
}
{
std::shared_lock lock(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Portal)]);
result.ProfilesIds_Portal.reserve(Profiles_Portal.size()*TableEntry<DefPortal>::ChunkSize);
counter = 0;
for(const auto& entry : Profiles_Portal)
for(const auto& item : entry->Entries) {
size_t id = counter++;
if(item)
result.ProfilesIds_Portal.push_back(id);
}
result.ProfilesIds_Portal.shrink_to_fit();
result.Profiles_Portal.reserve(result.ProfilesIds_Portal.size());
for(const auto& entry : Profiles_Portal)
for(const auto& item : entry->Entries)
if(item)
result.Profiles_Portal.push_back(&item.value());
}
{
std::shared_lock lock(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Entity)]);
result.ProfilesIds_Entity.reserve(Profiles_Entity.size()*TableEntry<DefEntity>::ChunkSize);
counter = 0;
for(const auto& entry : Profiles_Entity)
for(const auto& item : entry->Entries) {
size_t id = counter++;
if(item)
result.ProfilesIds_Entity.push_back(id);
}
result.ProfilesIds_Entity.shrink_to_fit();
result.Profiles_Entity.reserve(result.ProfilesIds_Entity.size());
for(const auto& entry : Profiles_Entity)
for(const auto& item : entry->Entries)
if(item)
result.Profiles_Entity.push_back(&item.value());
}
{
std::shared_lock lock(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Item)]);
result.ProfilesIds_Item.reserve(Profiles_Item.size()*TableEntry<DefItem>::ChunkSize);
counter = 0;
for(const auto& entry : Profiles_Item)
for(const auto& item : entry->Entries) {
size_t id = counter++;
if(item)
result.ProfilesIds_Item.push_back(id);
}
result.ProfilesIds_Item.shrink_to_fit();
result.Profiles_Item.reserve(result.ProfilesIds_Item.size());
for(const auto& entry : Profiles_Item)
for(const auto& item : entry->Entries)
if(item)
result.Profiles_Item.push_back(&item.value());
}
result.IdToDK = idToDK();
return result;
}
} }

View File

@@ -1,21 +1,64 @@
#pragma once #pragma once
#include "Common/Abstract.hpp" #include "Common/Abstract.hpp"
#include "Server/Abstract.hpp" #include "AssetsManager.hpp"
#include "Common/AssetsPreloader.hpp" #include "Common/IdProvider.hpp"
#include "Common/Net.hpp"
#include "TOSLib.hpp"
#include <array>
#include <mutex>
#include <sol/table.hpp> #include <sol/table.hpp>
#include <string_view>
#include <unordered_map> #include <unordered_map>
#include <unordered_set>
namespace LV::Server { namespace LV::Server {
struct DefVoxel_Base { }; struct ResourceBase {
struct DefNode_Base { }; std::string Domain, Key;
struct DefWorld_Base { }; };
struct DefPortal_Base { };
struct DefEntity_Base { }; class ContentManager;
struct DefItem_Base { };
struct DefVoxel : public ResourceBase {
std::u8string dumpToClient() const {
return {};
}
};
struct DefNode : public ResourceBase {
AssetsNodestate NodestateId;
std::u8string dumpToClient() const {
auto wr = TOS::ByteBuffer::Writer();
wr << uint32_t(NodestateId);
auto buff = wr.complite();
return (std::u8string) std::u8string_view((const char8_t*) buff.data(), buff.size());
}
};
struct DefWorld : public ResourceBase {
std::u8string dumpToClient() const {
return {};
}
};
struct DefPortal : public ResourceBase {
std::u8string dumpToClient() const {
return {};
}
};
struct DefEntity : public ResourceBase {
std::u8string dumpToClient() const {
return {};
}
};
struct DefItem : public ResourceBase {
std::u8string dumpToClient() const {
return {};
}
};
struct DefVoxel_Mod { }; struct DefVoxel_Mod { };
struct DefNode_Mod { }; struct DefNode_Mod { };
@@ -24,41 +67,370 @@ struct DefPortal_Mod { };
struct DefEntity_Mod { }; struct DefEntity_Mod { };
struct DefItem_Mod { }; struct DefItem_Mod { };
struct ResourceBase { struct DefVoxel_Base {
std::string Domain, Key; private:
friend ContentManager;
DefVoxel compile(AssetsManager& am, ContentManager& cm, const std::string_view domain, const std::string_view key, const std::vector<std::tuple<std::string, DefVoxel_Mod>>& mods) const {
return DefVoxel();
}
}; };
struct DefVoxel : public ResourceBase { }; struct DefNode_Base {
struct DefNode : public ResourceBase { private:
AssetsNodestate NodestateId; friend ContentManager;
std::vector<AssetsModel> ModelDeps; DefNode compile(AssetsManager& am, ContentManager& cm, const std::string_view domain, const std::string_view key, const std::vector<std::tuple<std::string, DefNode_Mod>>& mods) const {
std::vector<AssetsTexture> TextureDeps; DefNode profile;
std::string jsonKey = std::string(key)+".json";
profile.NodestateId = am.getId(EnumAssets::Nodestate, domain, jsonKey);
TOS::Logger("Compile").info() << domain << ' ' << key << " -> " << profile.NodestateId;
return profile;
}
}; };
struct DefWorld : public ResourceBase { };
struct DefPortal : public ResourceBase { };
struct DefEntity : public ResourceBase { };
struct DefItem : public ResourceBase { };
class ContentManager { struct DefWorld_Base {
private:
friend ContentManager;
DefWorld compile(AssetsManager& am, ContentManager& cm, const std::string_view domain, const std::string_view key, const std::vector<std::tuple<std::string, DefWorld_Mod>>& mods) const {
return DefWorld();
}
};
struct DefPortal_Base {
private:
friend ContentManager;
DefPortal compile(AssetsManager& am, ContentManager& cm, const std::string_view domain, const std::string_view key, const std::vector<std::tuple<std::string, DefPortal_Mod>>& mods) const {
return DefPortal();
}
};
struct DefEntity_Base {
private:
friend ContentManager;
DefEntity compile(AssetsManager& am, ContentManager& cm, const std::string_view domain, const std::string_view key, const std::vector<std::tuple<std::string, DefEntity_Mod>>& mods) const {
return DefEntity();
}
};
struct DefItem_Base {
private:
friend ContentManager;
DefItem compile(AssetsManager& am, ContentManager& cm, const std::string_view domain, const std::string_view key, const std::vector<std::tuple<std::string, DefItem_Mod>>& mods) const {
return DefItem();
}
};
/*
DK to id
id to profile
*/
class ContentManager : public IdProvider<EnumDefContent> {
public:
class LRU {
public:
LRU(ContentManager& cm)
: CM(&cm)
{
}
LRU(const LRU&) = default;
LRU(LRU&&) = default;
LRU& operator=(const LRU&) = default;
LRU& operator=(LRU&&) = default;
ResourceId getId(EnumDefContent type, const std::string_view domain, const std::string_view key) {
auto iter = DKToId[static_cast<size_t>(type)].find(BindDomainKeyViewInfo(domain, key));
if(iter == DKToId[static_cast<size_t>(type)].end()) {
ResourceId id = CM->getId(type, domain, key);
DKToId[static_cast<size_t>(type)].emplace_hint(iter, BindDomainKeyInfo((std::string) domain, (std::string) key), id);
return id;
}
return iter->second;
// switch(type) {
// case EnumDefContent::Voxel:
// case EnumDefContent::Node:
// case EnumDefContent::World:
// case EnumDefContent::Portal:
// case EnumDefContent::Entity:
// case EnumDefContent::Item:
// default:
// std::unreachable();
// }
}
ResourceId getIdVoxel(const std::string_view domain, const std::string_view key) {
return getId(EnumDefContent::Voxel, domain, key);
}
ResourceId getIdNode(const std::string_view domain, const std::string_view key) {
return getId(EnumDefContent::Node, domain, key);
}
ResourceId getIdWorld(const std::string_view domain, const std::string_view key) {
return getId(EnumDefContent::World, domain, key);
}
ResourceId getIdPortal(const std::string_view domain, const std::string_view key) {
return getId(EnumDefContent::Portal, domain, key);
}
ResourceId getIdEntity(const std::string_view domain, const std::string_view key) {
return getId(EnumDefContent::Entity, domain, key);
}
ResourceId getIdItem(const std::string_view domain, const std::string_view key) {
return getId(EnumDefContent::Item, domain, key);
}
private:
ContentManager* CM;
std::array<
ankerl::unordered_dense::map<BindDomainKeyInfo, ResourceId, KeyHash, KeyEq>,
MAX_ENUM
> DKToId;
std::unordered_map<DefVoxelId, std::optional<DefVoxel>*> Profiles_Voxel;
std::unordered_map<DefNodeId, std::optional<DefNode>*> Profiles_Node;
std::unordered_map<DefWorldId, std::optional<DefWorld>*> Profiles_World;
std::unordered_map<DefPortalId, std::optional<DefPortal>*> Profiles_Portal;
std::unordered_map<DefEntityId, std::optional<DefEntity>*> Profiles_Entity;
std::unordered_map<DefItemId, std::optional<DefItem>*> Profiles_Item;
};
public:
ContentManager(AssetsManager &am);
~ContentManager();
// Регистрирует определение контента
void registerBase(EnumDefContent type, const std::string& domain, const std::string& key, const sol::table& profile);
void unRegisterBase(EnumDefContent type, const std::string& domain, const std::string& key);
// Регистрация модификатора предмета модом
void registerModifier(EnumDefContent type, const std::string& mod, const std::string& domain, const std::string& key, const sol::table& profile);
void unRegisterModifier(EnumDefContent type, const std::string& mod, const std::string& domain, const std::string& key);
// Пометить все профили типа как изменённые (например, после перезагрузки ассетов)
// void markAllProfilesDirty(EnumDefContent type);
// Список всех зарегистрированных профилей выбранного типа
std::vector<ResourceId> collectProfileIds(EnumDefContent type) const;
// Компилирует изменённые профили
struct Out_buildEndProfiles {
std::vector<
std::tuple<DefVoxelId, const DefVoxel*>
> ChangedProfiles_Voxel;
std::vector<
std::tuple<DefNodeId, const DefNode*>
> ChangedProfiles_Node;
std::vector<
std::tuple<DefWorldId, const DefWorld*>
> ChangedProfiles_World;
std::vector<
std::tuple<DefPortalId, const DefPortal*>
> ChangedProfiles_Portal;
std::vector<
std::tuple<DefEntityId, const DefEntity*>
> ChangedProfiles_Entity;
std::vector<
std::tuple<DefItemId, const DefItem*>
> ChangedProfiles_Item;
std::array<
std::vector<ResourceId>,
MAX_ENUM
> LostProfiles;
std::array<
std::vector<BindDomainKeyInfo>,
static_cast<size_t>(EnumDefContent::MAX_ENUM)
> IdToDK;
};
// Компилирует конечные профили по базе и модификаторам (предоставляет клиентам изменённые и потерянные)
Out_buildEndProfiles buildEndProfiles();
struct Out_getAllProfiles {
std::vector<DefVoxelId> ProfilesIds_Voxel;
std::vector<const DefVoxel*> Profiles_Voxel;
std::vector<DefNodeId> ProfilesIds_Node;
std::vector<const DefNode*> Profiles_Node;
std::vector<DefWorldId> ProfilesIds_World;
std::vector<const DefWorld*> Profiles_World;
std::vector<DefPortalId> ProfilesIds_Portal;
std::vector<const DefPortal*> Profiles_Portal;
std::vector<DefEntityId> ProfilesIds_Entity;
std::vector<const DefEntity*> Profiles_Entity;
std::vector<DefItemId> ProfilesIds_Item;
std::vector<const DefItem*> Profiles_Item;
std::array<
std::vector<BindDomainKeyInfo>,
static_cast<size_t>(EnumDefContent::MAX_ENUM)
> IdToDK;
};
// Выдаёт все профили (для новых клиентов)
Out_getAllProfiles getAllProfiles();
std::optional<DefVoxel>& getEntry_Voxel(ResourceId resId) {
std::shared_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Voxel)]);
assert(resId / TableEntry<DefVoxel>::ChunkSize <= Profiles_Voxel.size());
return Profiles_Voxel[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];
}
std::optional<DefVoxel>& getEntry_Voxel(const std::string_view domain, const std::string_view key) {
return getEntry_Voxel(getId(EnumDefContent::Voxel, domain, key));
}
std::optional<DefNode>& getEntry_Node(ResourceId resId) {
std::shared_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Node)]);
assert(resId / TableEntry<DefNode>::ChunkSize < Profiles_Node.size());
return Profiles_Node[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];
}
std::optional<DefNode>& getEntry_Node(const std::string_view domain, const std::string_view key) {
return getEntry_Node(getId(EnumDefContent::Node, domain, key));
}
std::optional<DefWorld>& getEntry_World(ResourceId resId) {
std::shared_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::World)]);
assert(resId / TableEntry<DefWorld>::ChunkSize < Profiles_World.size());
return Profiles_World[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];
}
std::optional<DefWorld>& getEntry_World(const std::string_view domain, const std::string_view key) {
return getEntry_World(getId(EnumDefContent::World, domain, key));
}
std::optional<DefPortal>& getEntry_Portal(ResourceId resId) {
std::shared_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Portal)]);
assert(resId / TableEntry<DefPortal>::ChunkSize < Profiles_Portal.size());
return Profiles_Portal[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];
}
std::optional<DefPortal>& getEntry_Portal(const std::string_view domain, const std::string_view key) {
return getEntry_Portal(getId(EnumDefContent::Portal, domain, key));
}
std::optional<DefEntity>& getEntry_Entity(ResourceId resId) {
std::shared_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Entity)]);
assert(resId / TableEntry<DefEntity>::ChunkSize < Profiles_Entity.size());
return Profiles_Entity[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];
}
std::optional<DefEntity>& getEntry_Entity(const std::string_view domain, const std::string_view key) {
return getEntry_Entity(getId(EnumDefContent::Entity, domain, key));
}
std::optional<DefItem>& getEntry_Item(ResourceId resId) {
std::shared_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Item)]);
assert(resId / TableEntry<DefItem>::ChunkSize < Profiles_Item.size());
return Profiles_Item[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];
}
std::optional<DefItem>& getEntry_Item(const std::string_view domain, const std::string_view key) {
return getEntry_Item(getId(EnumDefContent::Item, domain, key));
}
ResourceId getId(EnumDefContent type, const std::string_view domain, const std::string_view key) {
ResourceId resId = IdProvider::getId(type, domain, key);
switch(type) {
case EnumDefContent::Voxel:
if(resId >= Profiles_Voxel.size()*TableEntry<DefVoxel>::ChunkSize) {
std::unique_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Voxel)]);
if(resId >= Profiles_Voxel.size()*TableEntry<DefVoxel>::ChunkSize)
Profiles_Voxel.push_back(std::make_unique<TableEntry<DefVoxel>>());
}
break;
case EnumDefContent::Node:
if(resId >= Profiles_Node.size()*TableEntry<DefNode>::ChunkSize) {
std::unique_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Node)]);
if(resId >= Profiles_Node.size()*TableEntry<DefNode>::ChunkSize)
Profiles_Node.push_back(std::make_unique<TableEntry<DefNode>>());
}
break;
case EnumDefContent::World:
if(resId >= Profiles_World.size()*TableEntry<DefWorld>::ChunkSize) {
std::unique_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::World)]);
if(resId >= Profiles_World.size()*TableEntry<DefWorld>::ChunkSize)
Profiles_World.push_back(std::make_unique<TableEntry<DefWorld>>());
}
break;
case EnumDefContent::Portal:
if(resId >= Profiles_Portal.size()*TableEntry<DefPortal>::ChunkSize) {
std::unique_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Portal)]);
if(resId >= Profiles_Portal.size()*TableEntry<DefPortal>::ChunkSize)
Profiles_Portal.push_back(std::make_unique<TableEntry<DefPortal>>());
}
break;
case EnumDefContent::Entity:
if(resId >= Profiles_Entity.size()*TableEntry<DefEntity>::ChunkSize) {
std::unique_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Entity)]);
if(resId >= Profiles_Entity.size()*TableEntry<DefEntity>::ChunkSize)
Profiles_Entity.push_back(std::make_unique<TableEntry<DefEntity>>());
}
break;
case EnumDefContent::Item:
if(resId >= Profiles_Item.size()*TableEntry<DefItem>::ChunkSize) {
std::unique_lock mtx(Profiles_Mtx[static_cast<size_t>(EnumDefContent::Item)]);
if(resId >= Profiles_Item.size()*TableEntry<DefItem>::ChunkSize)
Profiles_Item.push_back(std::make_unique<TableEntry<DefItem>>());
}
break;
default:
std::unreachable();
}
return resId;
}
LRU createLRU() {
return {*this};
}
private:
template<typename T> template<typename T>
struct TableEntry { struct TableEntry {
static constexpr size_t ChunkSize = 4096; static constexpr size_t ChunkSize = 4096;
std::array<std::optional<T>, ChunkSize> Entries; std::array<std::optional<T>, ChunkSize> Entries;
}; };
void registerBase_Node(ResourceId id, const std::string& domain, const std::string& key, const sol::table& profile);
void registerBase_World(ResourceId id, const std::string& domain, const std::string& key, const sol::table& profile);
void registerBase_Entity(ResourceId id, const std::string& domain, const std::string& key, const sol::table& profile);
// Следующие идентификаторы регистрации контента template<class type, class modType>
ResourceId NextId[(int) EnumDefContent::MAX_ENUM] = {}; void buildEndProfilesByType(auto& profiles, auto enumType, auto& base, auto& keys, auto& result, auto& mods);
// Домен -> {ключ -> идентификатор}
std::unordered_map<std::string, std::unordered_map<std::string, ResourceId>> ContentKeyToId[(int) EnumDefContent::MAX_ENUM]; TOS::Logger LOG = "Server>ContentManager";
AssetsManager& AM;
// Профили зарегистрированные модами // Профили зарегистрированные модами
std::vector<std::unique_ptr<TableEntry<DefVoxel>>> Profiles_Base_Voxel; std::vector<std::unique_ptr<TableEntry<DefVoxel_Base>>> Profiles_Base_Voxel;
std::vector<std::unique_ptr<TableEntry<DefNode>>> Profiles_Base_Node; std::vector<std::unique_ptr<TableEntry<DefNode_Base>>> Profiles_Base_Node;
std::vector<std::unique_ptr<TableEntry<DefWorld>>> Profiles_Base_World; std::vector<std::unique_ptr<TableEntry<DefWorld_Base>>> Profiles_Base_World;
std::vector<std::unique_ptr<TableEntry<DefPortal>>> Profiles_Base_Portal; std::vector<std::unique_ptr<TableEntry<DefPortal_Base>>> Profiles_Base_Portal;
std::vector<std::unique_ptr<TableEntry<DefEntity>>> Profiles_Base_Entity; std::vector<std::unique_ptr<TableEntry<DefEntity_Base>>> Profiles_Base_Entity;
std::vector<std::unique_ptr<TableEntry<DefItem>>> Profiles_Base_Item; std::vector<std::unique_ptr<TableEntry<DefItem_Base>>> Profiles_Base_Item;
// Изменения, накладываемые на профили // Изменения, накладываемые на профили
// Идентификатор [домен мода модификатора, модификатор] // Идентификатор [домен мода модификатора, модификатор]
@@ -71,151 +443,16 @@ class ContentManager {
// Затронутые профили в процессе регистраций // Затронутые профили в процессе регистраций
// По ним будут пересобраны профили // По ним будут пересобраны профили
std::vector<ResourceId> ProfileChanges[(int) EnumDefContent::MAX_ENUM]; std::vector<ResourceId> ProfileChanges[MAX_ENUM];
// Конечные профили контента // Конечные профили контента
std::array<std::shared_mutex, MAX_ENUM> Profiles_Mtx;
std::vector<std::unique_ptr<TableEntry<DefVoxel>>> Profiles_Voxel; std::vector<std::unique_ptr<TableEntry<DefVoxel>>> Profiles_Voxel;
std::vector<std::unique_ptr<TableEntry<DefNode>>> Profiles_Node; std::vector<std::unique_ptr<TableEntry<DefNode>>> Profiles_Node;
std::vector<std::unique_ptr<TableEntry<DefWorld>>> Profiles_World; std::vector<std::unique_ptr<TableEntry<DefWorld>>> Profiles_World;
std::vector<std::unique_ptr<TableEntry<DefPortal>>> Profiles_Portal; std::vector<std::unique_ptr<TableEntry<DefPortal>>> Profiles_Portal;
std::vector<std::unique_ptr<TableEntry<DefEntity>>> Profiles_Entity; std::vector<std::unique_ptr<TableEntry<DefEntity>>> Profiles_Entity;
std::vector<std::unique_ptr<TableEntry<DefItem>>> Profiles_Item; std::vector<std::unique_ptr<TableEntry<DefItem>>> Profiles_Item;
std::optional<DefVoxel>& getEntry_Voxel(ResourceId resId) { return Profiles_Voxel[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];}
std::optional<DefNode>& getEntry_Node(ResourceId resId) { return Profiles_Node[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];}
std::optional<DefWorld>& getEntry_World(ResourceId resId) { return Profiles_World[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];}
std::optional<DefPortal>& getEntry_Portal(ResourceId resId) { return Profiles_Portal[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];}
std::optional<DefEntity>& getEntry_Entity(ResourceId resId) { return Profiles_Entity[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];}
std::optional<DefItem>& getEntry_Item(ResourceId resId) { return Profiles_Item[resId / TableEntry<DefVoxel>::ChunkSize]->Entries[resId % TableEntry<DefVoxel>::ChunkSize];}
ResourceId getId(EnumDefContent type, const std::string& domain, const std::string& key) {
if(auto iterCKTI = ContentKeyToId[(int) type].find(domain); iterCKTI != ContentKeyToId[(int) type].end()) {
if(auto iterKey = iterCKTI->second.find(key); iterKey != iterCKTI->second.end()) {
return iterKey->second;
}
}
ResourceId resId = NextId[(int) type]++;
ContentKeyToId[(int) type][domain][key] = resId;
switch(type) {
case EnumDefContent::Voxel:
if(resId >= Profiles_Voxel.size()*TableEntry<DefVoxel>::ChunkSize)
Profiles_Voxel.push_back(std::make_unique<TableEntry<DefVoxel>>());
break;
case EnumDefContent::Node:
if(resId >= Profiles_Node.size()*TableEntry<DefNode>::ChunkSize)
Profiles_Node.push_back(std::make_unique<TableEntry<DefNode>>());
break;
case EnumDefContent::World:
if(resId >= Profiles_World.size()*TableEntry<DefWorld>::ChunkSize)
Profiles_World.push_back(std::make_unique<TableEntry<DefWorld>>());
break;
case EnumDefContent::Portal:
if(resId >= Profiles_Portal.size()*TableEntry<DefPortal>::ChunkSize)
Profiles_Portal.push_back(std::make_unique<TableEntry<DefPortal>>());
break;
case EnumDefContent::Entity:
if(resId >= Profiles_Entity.size()*TableEntry<DefEntity>::ChunkSize)
Profiles_Entity.push_back(std::make_unique<TableEntry<DefEntity>>());
break;
case EnumDefContent::Item:
if(resId >= Profiles_Item.size()*TableEntry<DefItem>::ChunkSize)
Profiles_Item.push_back(std::make_unique<TableEntry<DefItem>>());
break;
default:
std::unreachable();
}
return resId;
}
void registerBase_Node(ResourceId id, const std::string& domain, const std::string& key, const sol::table& profile);
void registerBase_World(ResourceId id, const std::string& domain, const std::string& key, const sol::table& profile);
void registerBase_Entity(ResourceId id, const std::string& domain, const std::string& key, const sol::table& profile);
public:
ContentManager(AssetsPreloader &am);
~ContentManager();
// Регистрирует определение контента
void registerBase(EnumDefContent type, const std::string& domain, const std::string& key, const sol::table& profile);
void unRegisterBase(EnumDefContent type, const std::string& domain, const std::string& key);
// Регистрация модификатора предмета модом
void registerModifier(EnumDefContent type, const std::string& mod, const std::string& domain, const std::string& key, const sol::table& profile);
void unRegisterModifier(EnumDefContent type, const std::string& mod, const std::string& domain, const std::string& key);
// Пометить все профили типа как изменённые (например, после перезагрузки ассетов)
void markAllProfilesDirty(EnumDefContent type);
// Список всех зарегистрированных профилей выбранного типа
std::vector<ResourceId> collectProfileIds(EnumDefContent type) const;
// Компилирует изменённые профили
struct Out_buildEndProfiles {
std::vector<ResourceId> ChangedProfiles[(int) EnumDefContent::MAX_ENUM];
};
Out_buildEndProfiles buildEndProfiles();
std::optional<DefVoxel*> getProfile_Voxel(ResourceId id) {
assert(id < Profiles_Voxel.size()*TableEntry<DefVoxel>::ChunkSize);
auto& value = Profiles_Voxel[id / TableEntry<DefVoxel>::ChunkSize]->Entries[id % TableEntry<DefVoxel>::ChunkSize];
if(value)
return {&*value};
else
return std::nullopt;
}
std::optional<DefNode*> getProfile_Node(ResourceId id) {
assert(id < Profiles_Node.size()*TableEntry<DefNode>::ChunkSize);
auto& value = Profiles_Node[id / TableEntry<DefNode>::ChunkSize]->Entries[id % TableEntry<DefNode>::ChunkSize];
if(value)
return {&*value};
else
return std::nullopt;
}
std::optional<DefWorld*> getProfile_World(ResourceId id) {
assert(id < Profiles_World.size()*TableEntry<DefWorld>::ChunkSize);
auto& value = Profiles_World[id / TableEntry<DefWorld>::ChunkSize]->Entries[id % TableEntry<DefWorld>::ChunkSize];
if(value)
return {&*value};
else
return std::nullopt;
}
std::optional<DefPortal*> getProfile_Portal(ResourceId id) {
assert(id < Profiles_Portal.size()*TableEntry<DefPortal>::ChunkSize);
auto& value = Profiles_Portal[id / TableEntry<DefPortal>::ChunkSize]->Entries[id % TableEntry<DefPortal>::ChunkSize];
if(value)
return {&*value};
else
return std::nullopt;
}
std::optional<DefEntity*> getProfile_Entity(ResourceId id) {
assert(id < Profiles_Entity.size()*TableEntry<DefEntity>::ChunkSize);
auto& value = Profiles_Entity[id / TableEntry<DefEntity>::ChunkSize]->Entries[id % TableEntry<DefEntity>::ChunkSize];
if(value)
return {&*value};
else
return std::nullopt;
}
std::optional<DefItem*> getProfile_Item(ResourceId id) {
assert(id < Profiles_Item.size()*TableEntry<DefItem>::ChunkSize);
auto& value = Profiles_Item[id / TableEntry<DefItem>::ChunkSize]->Entries[id % TableEntry<DefItem>::ChunkSize];
if(value)
return {&*value};
else
return std::nullopt;
}
ResourceId getContentId(EnumDefContent type, const std::string& domain, const std::string& key) {
return getId(type, domain, key);
}
private:
TOS::Logger LOG = "Server>ContentManager";
AssetsPreloader& AM;
}; };
} }

View File

@@ -44,6 +44,15 @@ namespace js = boost::json;
namespace LV::Server { namespace LV::Server {
template <typename T, size_t N>
bool hasAnyBindings(const std::array<std::vector<T>, N>& data) {
for(const auto& list : data) {
if(!list.empty())
return true;
}
return false;
}
std::string ModInfo::dump() const { std::string ModInfo::dump() const {
js::object obj; js::object obj;
@@ -478,15 +487,14 @@ std::variant<std::vector<ModInfo>, std::vector<std::string>> resolveDepends(cons
GameServer::GameServer(asio::io_context &ioc, fs::path worldPath) GameServer::GameServer(asio::io_context &ioc, fs::path worldPath)
: AsyncObject(ioc), : AsyncObject(ioc),
Content(ioc) Content(ioc), BackingAsyncLua(Content.CM)
{ {
init(worldPath); init(worldPath);
} }
GameServer::~GameServer() { GameServer::~GameServer() {
shutdown("on ~GameServer"); shutdown("on ~GameServer");
BackingChunkPressure.NeedShutdown = true; BackingChunkPressure.NeedShutdown.store(true, std::memory_order_release);
BackingChunkPressure.Symaphore.notify_all();
BackingNoiseGenerator.NeedShutdown = true; BackingNoiseGenerator.NeedShutdown = true;
BackingAsyncLua.NeedShutdown = true; BackingAsyncLua.NeedShutdown = true;
@@ -502,27 +510,19 @@ GameServer::~GameServer() {
} }
void GameServer::BackingChunkPressure_t::run(int id) { void GameServer::BackingChunkPressure_t::run(int id) {
// static thread_local int local_counter = -1;
int iteration = 0;
LOG.debug() << "Старт потока " << id; LOG.debug() << "Старт потока " << id;
try { try {
while(true) { while(true) {
// local_counter++; if(NeedShutdown.load(std::memory_order_acquire)) {
// LOG.debug() << "Ожидаю начала " << id << ' ' << local_counter; CollectStart->arrive_and_drop();
{ LOG.debug() << "Завершение выполнения потока " << id;
std::unique_lock<std::mutex> lock(Mutex); break;
Symaphore.wait(lock, [&](){ return iteration != Iteration || NeedShutdown; });
if(NeedShutdown) {
LOG.debug() << "Завершение выполнения потока " << id;
break;
}
iteration = Iteration;
} }
assert(RunCollect > 0); CollectStart->arrive_and_wait();
assert(RunCompress > 0);
bool shutting = NeedShutdown.load(std::memory_order_acquire);
// Сбор данных // Сбор данных
size_t pullSize = Threads.size(); size_t pullSize = Threads.size();
@@ -537,157 +537,159 @@ void GameServer::BackingChunkPressure_t::run(int id) {
std::vector<std::pair<WorldId_t, std::vector<std::pair<Pos::GlobalRegion, Dump>>>> dump; std::vector<std::pair<WorldId_t, std::vector<std::pair<Pos::GlobalRegion, Dump>>>> dump;
for(const auto& [worldId, world] : *Worlds) { if(!shutting) {
const auto &worldObj = *world; try {
std::vector<std::pair<Pos::GlobalRegion, Dump>> dumpWorld; for(const auto& [worldId, world] : *Worlds) {
const auto &worldObj = *world;
std::vector<std::pair<Pos::GlobalRegion, Dump>> dumpWorld;
for(const auto& [regionPos, region] : worldObj.Regions) { for(const auto& [regionPos, region] : worldObj.Regions) {
auto& regionObj = *region; auto& regionObj = *region;
if(counter++ % pullSize != id) { if(counter++ % pullSize != id) {
continue; continue;
}
Dump dumpRegion;
dumpRegion.CECs = regionObj.RMs;
dumpRegion.IsChunkChanged_Voxels = regionObj.IsChunkChanged_Voxels;
regionObj.IsChunkChanged_Voxels = 0;
dumpRegion.IsChunkChanged_Nodes = regionObj.IsChunkChanged_Nodes;
regionObj.IsChunkChanged_Nodes = 0;
if(!regionObj.NewRMs.empty()) {
dumpRegion.NewCECs = std::move(regionObj.NewRMs);
dumpRegion.Voxels = regionObj.Voxels;
for(int z = 0; z < 4; z++)
for(int y = 0; y < 4; y++)
for(int x = 0; x < 4; x++)
{
auto &toPtr = dumpRegion.Nodes[Pos::bvec4u(x, y, z)];
const Node *fromPtr = regionObj.Nodes[Pos::bvec4u(x, y, z).pack()].data();
std::copy(fromPtr, fromPtr+16*16*16, toPtr.data());
} }
} else {
if(dumpRegion.IsChunkChanged_Voxels) {
for(int index = 0; index < 64; index++) {
if(((dumpRegion.IsChunkChanged_Voxels >> index) & 0x1) == 0)
continue;
Pos::bvec4u chunkPos; Dump dumpRegion;
chunkPos.unpack(index);
auto voxelIter = regionObj.Voxels.find(chunkPos); dumpRegion.CECs = regionObj.RMs;
if(voxelIter != regionObj.Voxels.end()) { dumpRegion.IsChunkChanged_Voxels = regionObj.IsChunkChanged_Voxels;
dumpRegion.Voxels[chunkPos] = voxelIter->second; regionObj.IsChunkChanged_Voxels = 0;
} else { dumpRegion.IsChunkChanged_Nodes = regionObj.IsChunkChanged_Nodes;
dumpRegion.Voxels[chunkPos] = {}; regionObj.IsChunkChanged_Nodes = 0;
if(!regionObj.NewRMs.empty()) {
dumpRegion.NewCECs = std::move(regionObj.NewRMs);
dumpRegion.Voxels = regionObj.Voxels;
for(int z = 0; z < 4; z++)
for(int y = 0; y < 4; y++)
for(int x = 0; x < 4; x++)
{
auto &toPtr = dumpRegion.Nodes[Pos::bvec4u(x, y, z)];
const Node *fromPtr = regionObj.Nodes[Pos::bvec4u(x, y, z).pack()].data();
std::copy(fromPtr, fromPtr+16*16*16, toPtr.data());
}
} else {
if(dumpRegion.IsChunkChanged_Voxels) {
for(int index = 0; index < 64; index++) {
if(((dumpRegion.IsChunkChanged_Voxels >> index) & 0x1) == 0)
continue;
Pos::bvec4u chunkPos;
chunkPos.unpack(index);
auto voxelIter = regionObj.Voxels.find(chunkPos);
if(voxelIter != regionObj.Voxels.end()) {
dumpRegion.Voxels[chunkPos] = voxelIter->second;
} else {
dumpRegion.Voxels[chunkPos] = {};
}
}
}
if(dumpRegion.IsChunkChanged_Nodes) {
for(int index = 0; index < 64; index++) {
if(((dumpRegion.IsChunkChanged_Nodes >> index) & 0x1) == 0)
continue;
Pos::bvec4u chunkPos;
chunkPos.unpack(index);
auto &toPtr = dumpRegion.Nodes[chunkPos];
const Node *fromPtr = regionObj.Nodes[chunkPos.pack()].data();
std::copy(fromPtr, fromPtr+16*16*16, toPtr.data());
}
} }
} }
}
if(dumpRegion.IsChunkChanged_Nodes) { if(!dumpRegion.CECs.empty()) {
for(int index = 0; index < 64; index++) { dumpWorld.push_back({regionPos, std::move(dumpRegion)});
if(((dumpRegion.IsChunkChanged_Nodes >> index) & 0x1) == 0)
continue;
Pos::bvec4u chunkPos;
chunkPos.unpack(index);
auto &toPtr = dumpRegion.Nodes[chunkPos];
const Node *fromPtr = regionObj.Nodes[chunkPos.pack()].data();
std::copy(fromPtr, fromPtr+16*16*16, toPtr.data());
} }
} }
}
if(!dumpRegion.CECs.empty()) { if(!dumpWorld.empty()) {
dumpWorld.push_back({regionPos, std::move(dumpRegion)}); dump.push_back({worldId, std::move(dumpWorld)});
}
} }
} } catch(const std::exception&) {
NeedShutdown.store(true, std::memory_order_release);
if(!dumpWorld.empty()) { shutting = true;
dump.push_back({worldId, std::move(dumpWorld)});
} }
} }
// Синхронизация CollectEnd->arrive_and_wait();
// LOG.debug() << "Синхронизирую " << id << ' ' << local_counter;
{
std::unique_lock<std::mutex> lock(Mutex);
RunCollect -= 1;
Symaphore.notify_all();
}
// Сжатие и отправка игрокам // Сжатие и отправка игрокам
for(auto& [worldId, world] : dump) { if(!shutting) {
for(auto& [regionPos, region] : world) { try {
for(auto& [chunkPos, chunk] : region.Voxels) { for(auto& [worldId, world] : dump) {
std::u8string cmp = compressVoxels(chunk); for(auto& [regionPos, region] : world) {
for(auto& [chunkPos, chunk] : region.Voxels) {
std::u8string cmp = compressVoxels(chunk);
for(auto& ptr : region.NewCECs) { for(auto& ptr : region.NewCECs) {
ptr->prepareChunkUpdate_Voxels(worldId, regionPos, chunkPos, cmp); ptr->prepareChunkUpdate_Voxels(worldId, regionPos, chunkPos, cmp);
}
if((region.IsChunkChanged_Voxels >> chunkPos.pack()) & 0x1) {
for(auto& ptr : region.CECs) {
bool skip = false;
for(auto& ptr2 : region.NewCECs) {
if(ptr == ptr2) {
skip = true;
break;
}
} }
if(skip) if((region.IsChunkChanged_Voxels >> chunkPos.pack()) & 0x1) {
continue; for(auto& ptr : region.CECs) {
bool skip = false;
for(auto& ptr2 : region.NewCECs) {
if(ptr == ptr2) {
skip = true;
break;
}
}
ptr->prepareChunkUpdate_Voxels(worldId, regionPos, chunkPos, cmp); if(skip)
} continue;
}
} ptr->prepareChunkUpdate_Voxels(worldId, regionPos, chunkPos, cmp);
}
for(auto& [chunkPos, chunk] : region.Nodes) { }
std::u8string cmp = compressNodes(chunk.data()); }
for(auto& ptr : region.NewCECs) { for(auto& [chunkPos, chunk] : region.Nodes) {
ptr->prepareChunkUpdate_Nodes(worldId, regionPos, chunkPos, cmp); std::u8string cmp = compressNodes(chunk.data());
}
for(auto& ptr : region.NewCECs) {
if((region.IsChunkChanged_Nodes >> chunkPos.pack()) & 0x1) { ptr->prepareChunkUpdate_Nodes(worldId, regionPos, chunkPos, cmp);
for(auto& ptr : region.CECs) { }
bool skip = false;
for(auto& ptr2 : region.NewCECs) { if((region.IsChunkChanged_Nodes >> chunkPos.pack()) & 0x1) {
if(ptr == ptr2) { for(auto& ptr : region.CECs) {
skip = true; bool skip = false;
break; for(auto& ptr2 : region.NewCECs) {
} if(ptr == ptr2) {
} skip = true;
break;
if(skip) }
continue; }
ptr->prepareChunkUpdate_Nodes(worldId, regionPos, chunkPos, cmp); if(skip)
continue;
ptr->prepareChunkUpdate_Nodes(worldId, regionPos, chunkPos, cmp);
}
}
} }
} }
} }
} catch(const std::exception&) {
NeedShutdown.store(true, std::memory_order_release);
shutting = true;
} }
} }
// Синхронизация CompressEnd->arrive_and_wait();
// LOG.debug() << "Конец " << id << ' ' << local_counter;
{ if(shutting)
std::unique_lock<std::mutex> lock(Mutex); continue;
RunCompress -= 1;
Symaphore.notify_all();
}
} }
} catch(const std::exception& exc) { } catch(const std::exception& exc) {
std::unique_lock<std::mutex> lock(Mutex); NeedShutdown.store(true, std::memory_order_release);
NeedShutdown = true;
LOG.error() << "Ошибка выполнения потока " << id << ":\n" << exc.what(); LOG.error() << "Ошибка выполнения потока " << id << ":\n" << exc.what();
} }
Symaphore.notify_all();
} }
void GameServer::BackingNoiseGenerator_t::run(int id) { void GameServer::BackingNoiseGenerator_t::run(int id) {
@@ -769,17 +771,19 @@ void GameServer::BackingAsyncLua_t::run(int id) {
out.Voxels.clear(); out.Voxels.clear();
out.Entityes.clear(); out.Entityes.clear();
auto lru = CM.createLRU();
{ {
constexpr DefNodeId kNodeAir = 0; DefNodeId kNodeAir = 0;
constexpr DefNodeId kNodeGrass = 2; DefNodeId kNodeGrass = lru.getIdNode("test", "grass");
constexpr uint8_t kMetaGrass = 1; uint8_t kMetaGrass = 1;
constexpr DefNodeId kNodeDirt = 3; DefNodeId kNodeDirt = lru.getIdNode("test", "dirt");
constexpr DefNodeId kNodeStone = 4; DefNodeId kNodeStone = lru.getIdNode("test", "stone");
constexpr DefNodeId kNodeWood = 1; DefNodeId kNodeWood = lru.getIdNode("test", "log");
constexpr DefNodeId kNodeLeaves = 5; DefNodeId kNodeLeaves = lru.getIdNode("test", "leaves");
constexpr DefNodeId kNodeLava = 7; DefNodeId kNodeLava = lru.getIdNode("test", "lava");
constexpr DefNodeId kNodeWater = 8; DefNodeId kNodeWater = lru.getIdNode("test", "water");
constexpr DefNodeId kNodeFire = 9; DefNodeId kNodeFire = lru.getIdNode("test", "fire");
auto hash32 = [](uint32_t x) { auto hash32 = [](uint32_t x) {
x ^= x >> 16; x ^= x >> 16;
@@ -914,9 +918,9 @@ void GameServer::BackingAsyncLua_t::run(int id) {
constexpr int kTestGlobalY = 64; constexpr int kTestGlobalY = 64;
if(regionBase.y <= kTestGlobalY && (regionBase.y + 63) >= kTestGlobalY) { if(regionBase.y <= kTestGlobalY && (regionBase.y + 63) >= kTestGlobalY) {
int localY = kTestGlobalY - regionBase.y; int localY = kTestGlobalY - regionBase.y;
setNode(2, localY, 2, kNodeLava, 0, false); setNode(7, localY, 2, kNodeLava, 0, false);
setNode(4, localY, 2, kNodeWater, 0, false); setNode(8, localY, 2, kNodeWater, 0, false);
setNode(6, localY, 2, kNodeFire, 0, false); setNode(9, localY, 2, kNodeFire, 0, false);
} }
} }
} }
@@ -1336,7 +1340,8 @@ void GameServer::init(fs::path worldPath) {
AssetsInit.Assets.push_back(mlt.LoadChain[index].Path / "assets"); AssetsInit.Assets.push_back(mlt.LoadChain[index].Path / "assets");
} }
Content.AM.applyResourceChange(Content.AM.reloadResources(AssetsInit)); auto capru = Content.AM.checkAndPrepareResourcesUpdate(AssetsInit);
Content.AM.applyResourcesUpdate(capru);
LOG.info() << "Пре Инициализация"; LOG.info() << "Пре Инициализация";
@@ -1345,7 +1350,7 @@ void GameServer::init(fs::path worldPath) {
// Content.CM.registerBase(EnumDefContent::Node, "core", "none", t); // Content.CM.registerBase(EnumDefContent::Node, "core", "none", t);
Content.CM.registerBase(EnumDefContent::World, "test", "devel_world", t); Content.CM.registerBase(EnumDefContent::World, "test", "devel_world", t);
Content.CM.registerBase(EnumDefContent::Entity, "core", "player", t); Content.CM.registerBase(EnumDefContent::Entity, "core", "player", t);
PlayerEntityDefId = Content.CM.getContentId(EnumDefContent::Entity, "core", "player"); // PlayerEntityDefId = Content.CM.getContentId(EnumDefContent::Entity, "core", "player");
} }
initLuaPre(); initLuaPre();
@@ -1358,7 +1363,6 @@ void GameServer::init(fs::path worldPath) {
Content.CM.buildEndProfiles(); Content.CM.buildEndProfiles();
LOG.info() << "Инициализация"; LOG.info() << "Инициализация";
initLua(); initLua();
pushEvent("init"); pushEvent("init");
@@ -1379,6 +1383,7 @@ void GameServer::init(fs::path worldPath) {
LOG.info() << "Загрузка существующих миров..."; LOG.info() << "Загрузка существующих миров...";
BackingChunkPressure.Threads.resize(4); BackingChunkPressure.Threads.resize(4);
BackingChunkPressure.Worlds = &Expanse.Worlds; BackingChunkPressure.Worlds = &Expanse.Worlds;
BackingChunkPressure.init(BackingChunkPressure.Threads.size());
for(size_t iter = 0; iter < BackingChunkPressure.Threads.size(); iter++) { for(size_t iter = 0; iter < BackingChunkPressure.Threads.size(); iter++) {
BackingChunkPressure.Threads[iter] = std::thread(&BackingChunkPressure_t::run, &BackingChunkPressure, iter); BackingChunkPressure.Threads[iter] = std::thread(&BackingChunkPressure_t::run, &BackingChunkPressure, iter);
} }
@@ -1401,7 +1406,10 @@ void GameServer::prerun() {
auto useLock = UseLock.lock(); auto useLock = UseLock.lock();
run(); run();
} catch(const std::exception& exc) {
LOG.error() << "Исключение в GameServer::run: " << exc.what();
} catch(...) { } catch(...) {
LOG.error() << "Неизвестное исключение в GameServer::run";
} }
IsAlive = false; IsAlive = false;
@@ -1420,6 +1428,7 @@ void GameServer::run() {
while(true) { while(true) {
((uint32_t&) Game.AfterStartTime) += (uint32_t) (CurrentTickDuration*256); ((uint32_t&) Game.AfterStartTime) += (uint32_t) (CurrentTickDuration*256);
Game.Tick++;
std::chrono::steady_clock::time_point atTickStart = std::chrono::steady_clock::now(); std::chrono::steady_clock::time_point atTickStart = std::chrono::steady_clock::now();
@@ -1451,7 +1460,14 @@ void GameServer::run() {
stepConnections(); stepConnections();
stepModInitializations(); stepModInitializations();
IWorldSaveBackend::TickSyncInfo_Out dat1 = stepDatabaseSync(); IWorldSaveBackend::TickSyncInfo_Out dat1;
try {
dat1 = stepDatabaseSync();
} catch(const std::exception& exc) {
LOG.error() << "Ошибка stepDatabaseSync: " << exc.what();
} catch(...) {
LOG.error() << "Неизвестная ошибка stepDatabaseSync";
}
stepGeneratorAndLuaAsync(std::move(dat1)); stepGeneratorAndLuaAsync(std::move(dat1));
stepPlayerProceed(); stepPlayerProceed();
stepWorldPhysic(); stepWorldPhysic();
@@ -1597,11 +1613,12 @@ void GameServer::stepConnections() {
} }
if(!newClients.empty()) { if(!newClients.empty()) {
AssetsPreloader::Out_fullSync fullSync = Content.AM.collectFullSync();
std::array<std::vector<ResourceId>, static_cast<size_t>(EnumAssets::MAX_ENUM)> lost{}; std::array<std::vector<ResourceId>, static_cast<size_t>(EnumAssets::MAX_ENUM)> lost{};
std::vector<Net::Packet> packets = RemoteClient::makePackets_informateAssets_DK(fullSync.IdToDK); std::vector<Net::Packet> packets;
packets.push_back(RemoteClient::makePacket_informateAssets_HH(fullSync.HashHeaders, lost)); packets.push_back(RemoteClient::makePacket_informateAssets_DK(Content.AM.idToDK()));
packets.push_back(RemoteClient::makePacket_informateAssets_HH(Content.AM.collectHashBindings(), lost));
packets.append_range(RemoteClient::makePackets_informateDefContent_Full(Content.CM.getAllProfiles()));
for(const std::shared_ptr<RemoteClient>& client : newClients) { for(const std::shared_ptr<RemoteClient>& client : newClients) {
if(!packets.empty()) { if(!packets.empty()) {
@@ -1672,30 +1689,34 @@ void GameServer::reloadMods() {
{ {
// TODO: перезагрузка модов // TODO: перезагрузка модов
Content.CM.buildEndProfiles(); ContentManager::Out_buildEndProfiles out = Content.CM.buildEndProfiles();
packetsToSend.append_range(RemoteClient::makePackets_informateDefContentUpdate(out));
} }
LOG.info() << "Перезагрузка ассетов"; LOG.info() << "Перезагрузка ассетов";
{ {
{ {
AssetsPreloader::Out_applyResourceChange applied AssetsManager::Out_checkAndPrepareResourcesUpdate capru = Content.AM.checkAndPrepareResourcesUpdate(AssetsInit);
= Content.AM.applyResourceChange(Content.AM.reloadResources(AssetsInit)); AssetsManager::Out_applyResourcesUpdate aru = Content.AM.applyResourcesUpdate(capru);
if(!applied.NewOrChange.empty() || !applied.Lost.empty()) if(!capru.ResourceUpdates.empty() || !capru.LostLinks.empty())
packetsToSend.push_back( packetsToSend.push_back(
RemoteClient::makePacket_informateAssets_HH( RemoteClient::makePacket_informateAssets_HH(
applied.NewOrChange, aru.NewOrUpdates,
applied.Lost capru.LostLinks
) )
); );
} }
{ {
AssetsPreloader::Out_bakeId baked = Content.AM.bakeIdTables(); std::array<
if(!baked.IdToDK.empty()) { std::vector<AssetsManager::BindDomainKeyInfo>,
std::vector<Net::Packet> dkPackets = RemoteClient::makePackets_informateAssets_DK(baked.IdToDK); static_cast<size_t>(EnumAssets::MAX_ENUM)
packetsToSend.insert(packetsToSend.end(), dkPackets.begin(), dkPackets.end()); > baked = Content.AM.bake();
if(hasAnyBindings(baked)) {
packetsToSend.push_back(RemoteClient::makePacket_informateAssets_DK(baked));
} }
} }
} }
@@ -1710,73 +1731,202 @@ void GameServer::reloadMods() {
IWorldSaveBackend::TickSyncInfo_Out GameServer::stepDatabaseSync() { IWorldSaveBackend::TickSyncInfo_Out GameServer::stepDatabaseSync() {
IWorldSaveBackend::TickSyncInfo_In toDB; IWorldSaveBackend::TickSyncInfo_In toDB;
for(std::shared_ptr<RemoteClient>& remoteClient : Game.RemoteClients) { constexpr uint32_t kRegionUnloadDelayTicks = 300;
assert(remoteClient); constexpr uint8_t kUnloadHysteresisExtraRegions = 1;
// Пересчитать зоны наблюдения const uint32_t nowTick = Game.Tick;
if(remoteClient->CrossedRegion) {
remoteClient->CrossedRegion = false;
// Пересчёт зон наблюдения
std::vector<ContentViewCircle> newCVCs;
{ for(std::shared_ptr<RemoteClient>& remoteClient : Game.RemoteClients) {
std::vector<std::tuple<WorldId_t, Pos::Object, uint8_t>> points = remoteClient->getViewPoints(); assert(remoteClient);
for(auto& [wId, pos, radius] : points) {
assert(radius < 5); // 1) Если игрок пересёк границу региона — пересчитываем области наблюдения.
// Вводим гистерезис: загрузка по "внутренней" границе, выгрузка по "внешней" (+1 регион).
if(remoteClient->CrossedRegion) {
remoteClient->CrossedRegion = false;
std::vector<ContentViewCircle> innerCVCs;
std::vector<ContentViewCircle> outerCVCs;
{
std::vector<std::tuple<WorldId_t, Pos::Object, uint8_t>> points = remoteClient->getViewPoints();
for(auto& [wId, pos, radius] : points) {
assert(radius < 5);
// Внутренняя область (на загрузку)
{
ContentViewCircle cvc; ContentViewCircle cvc;
cvc.WorldId = wId; cvc.WorldId = wId;
cvc.Pos = Pos::Object_t::asRegionsPos(pos); cvc.Pos = Pos::Object_t::asRegionsPos(pos);
cvc.Range = radius*radius; cvc.Range = int16_t(radius * radius);
std::vector<ContentViewCircle> list = Expanse.accumulateContentViewCircles(cvc); std::vector<ContentViewCircle> list = Expanse.accumulateContentViewCircles(cvc);
newCVCs.insert(newCVCs.end(), list.begin(), list.end()); innerCVCs.insert(innerCVCs.end(), list.begin(), list.end());
}
// Внешняя область (на удержание/выгрузку) = внутренняя + 1 регион
{
uint8_t outerRadius = radius + kUnloadHysteresisExtraRegions;
if(outerRadius > 5) outerRadius = 5;
ContentViewCircle cvc;
cvc.WorldId = wId;
cvc.Pos = Pos::Object_t::asRegionsPos(pos);
cvc.Range = int16_t(outerRadius * outerRadius);
std::vector<ContentViewCircle> list = Expanse.accumulateContentViewCircles(cvc);
outerCVCs.insert(outerCVCs.end(), list.begin(), list.end());
} }
} }
}
ContentViewInfo newCbg = Expanse_t::makeContentViewInfo(newCVCs); ContentViewInfo viewInner = Expanse_t::makeContentViewInfo(innerCVCs);
ContentViewInfo viewOuter = Expanse_t::makeContentViewInfo(outerCVCs);
ContentViewInfo_Diff diff = newCbg.diffWith(remoteClient->ContentViewState); // Отменяем отложенную выгрузку для регионов, которые снова попали во внешнюю область
if(!diff.WorldsNew.empty()) { for(const auto& [worldId, regions] : viewOuter.Regions) {
// Сообщить о новых мирах auto itWorld = remoteClient->PendingRegionUnload.find(worldId);
for(const WorldId_t id : diff.WorldsNew) { if(itWorld == remoteClient->PendingRegionUnload.end())
auto iter = Expanse.Worlds.find(id); continue;
assert(iter != Expanse.Worlds.end());
remoteClient->prepareWorldUpdate(id, iter->second.get()); for(const Pos::GlobalRegion& pos : regions) {
} itWorld->second.erase(pos);
} }
remoteClient->ContentViewState = newCbg; if(itWorld->second.empty())
// Вычистка не наблюдаемых регионов remoteClient->PendingRegionUnload.erase(itWorld);
for(const auto& [worldId, regions] : diff.RegionsLost) }
remoteClient->prepareRegionsRemove(worldId, regions);
// и миров
for(const WorldId_t worldId : diff.WorldsLost)
remoteClient->prepareWorldRemove(worldId);
// Подписываем игрока на наблюдение за регионами // Загрузка: только по внутренней границе
for(const auto& [worldId, regions] : diff.RegionsNew) { ContentViewInfo_Diff diffInner = viewInner.diffWith(remoteClient->ContentViewState);
auto iterWorld = Expanse.Worlds.find(worldId);
assert(iterWorld != Expanse.Worlds.end());
std::vector<Pos::GlobalRegion> notLoaded = iterWorld->second->onRemoteClient_RegionsEnter(worldId, remoteClient, regions); if(!diffInner.WorldsNew.empty()) {
if(!notLoaded.empty()) { // Сообщить о новых мирах
// Добавляем к списку на загрузку for(const WorldId_t id : diffInner.WorldsNew) {
std::vector<Pos::GlobalRegion> &tl = toDB.Load[worldId]; auto iter = Expanse.Worlds.find(id);
tl.insert(tl.end(), notLoaded.begin(), notLoaded.end()); assert(iter != Expanse.Worlds.end());
}
remoteClient->prepareWorldUpdate(id, iter->second.get());
}
}
// Подписываем игрока на наблюдение за регионами (внутренняя область)
for(const auto& [worldId, regions] : diffInner.RegionsNew) {
// Добавляем в состояние клиента (слиянием, т.к. там могут быть регионы на удержании)
{
auto& cur = remoteClient->ContentViewState.Regions[worldId];
std::vector<Pos::GlobalRegion> merged;
merged.reserve(cur.size() + regions.size());
std::merge(cur.begin(), cur.end(), regions.begin(), regions.end(), std::back_inserter(merged));
merged.erase(std::unique(merged.begin(), merged.end()), merged.end());
cur = std::move(merged);
} }
// Отписываем то, что игрок больше не наблюдает auto iterWorld = Expanse.Worlds.find(worldId);
for(const auto& [worldId, regions] : diff.RegionsLost) { assert(iterWorld != Expanse.Worlds.end());
auto iterWorld = Expanse.Worlds.find(worldId);
assert(iterWorld != Expanse.Worlds.end());
iterWorld->second->onRemoteClient_RegionsLost(worldId, remoteClient, regions); std::vector<Pos::GlobalRegion> notLoaded = iterWorld->second->onRemoteClient_RegionsEnter(worldId, remoteClient, regions);
if(!notLoaded.empty()) {
// Добавляем к списку на загрузку
std::vector<Pos::GlobalRegion> &tl = toDB.Load[worldId];
tl.insert(tl.end(), notLoaded.begin(), notLoaded.end());
}
}
// Кандидаты на выгрузку: то, что есть у клиента, но не попадает во внешнюю область (гистерезис)
for(const auto& [worldId, curRegions] : remoteClient->ContentViewState.Regions) {
std::vector<Pos::GlobalRegion> outer;
auto itOuter = viewOuter.Regions.find(worldId);
if(itOuter != viewOuter.Regions.end())
outer = itOuter->second;
std::vector<Pos::GlobalRegion> toDelay;
toDelay.reserve(curRegions.size());
if(outer.empty()) {
toDelay = curRegions;
} else {
std::set_difference(
curRegions.begin(), curRegions.end(),
outer.begin(), outer.end(),
std::back_inserter(toDelay)
);
}
if(!toDelay.empty()) {
auto& pending = remoteClient->PendingRegionUnload[worldId];
for(const Pos::GlobalRegion& pos : toDelay) {
// если уже ждёт выгрузки — не трогаем
if(pending.find(pos) == pending.end())
pending[pos] = nowTick + kRegionUnloadDelayTicks;
}
} }
} }
} }
// 2) Отложенная выгрузка: если истекла задержка — реально удаляем регион из зоны видимости
if(!remoteClient->PendingRegionUnload.empty()) {
std::unordered_map<WorldId_t, std::vector<Pos::GlobalRegion>> expiredByWorld;
for(auto itWorld = remoteClient->PendingRegionUnload.begin(); itWorld != remoteClient->PendingRegionUnload.end(); ) {
const WorldId_t worldId = itWorld->first;
auto& regMap = itWorld->second;
std::vector<Pos::GlobalRegion> expired;
for(auto itReg = regMap.begin(); itReg != regMap.end(); ) {
if(itReg->second <= nowTick) {
expired.push_back(itReg->first);
itReg = regMap.erase(itReg);
} else {
++itReg;
}
}
if(!expired.empty()) {
std::sort(expired.begin(), expired.end());
expiredByWorld[worldId] = std::move(expired);
}
if(regMap.empty())
itWorld = remoteClient->PendingRegionUnload.erase(itWorld);
else
++itWorld;
}
// Применяем выгрузку: отписка + сообщение клиенту + актуализация ContentViewState
for(auto& [worldId, expired] : expiredByWorld) {
// Удаляем регионы из состояния клиента
auto itCur = remoteClient->ContentViewState.Regions.find(worldId);
if(itCur != remoteClient->ContentViewState.Regions.end()) {
std::vector<Pos::GlobalRegion> kept;
kept.reserve(itCur->second.size());
std::set_difference(
itCur->second.begin(), itCur->second.end(),
expired.begin(), expired.end(),
std::back_inserter(kept)
);
itCur->second = std::move(kept);
}
// Сообщаем клиенту и мирам
remoteClient->prepareRegionsRemove(worldId, expired);
auto iterWorld = Expanse.Worlds.find(worldId);
if(iterWorld != Expanse.Worlds.end()) {
iterWorld->second->onRemoteClient_RegionsLost(worldId, remoteClient, expired);
}
// Если в мире больше нет наблюдаемых регионов — удалить мир у клиента
auto itStateWorld = remoteClient->ContentViewState.Regions.find(worldId);
if(itStateWorld != remoteClient->ContentViewState.Regions.end() && itStateWorld->second.empty()) {
remoteClient->ContentViewState.Regions.erase(itStateWorld);
remoteClient->prepareWorldRemove(worldId);
}
}
}
}
for(auto& [worldId, regions] : toDB.Load) { for(auto& [worldId, regions] : toDB.Load) {
std::sort(regions.begin(), regions.end()); std::sort(regions.begin(), regions.end());
auto eraseIter = std::unique(regions.begin(), regions.end()); auto eraseIter = std::unique(regions.begin(), regions.end());
@@ -1786,7 +1936,7 @@ IWorldSaveBackend::TickSyncInfo_Out GameServer::stepDatabaseSync() {
// Обзавелись списком на прогрузку регионов // Обзавелись списком на прогрузку регионов
// Теперь узнаем что нужно сохранить и что из регионов было выгружено // Теперь узнаем что нужно сохранить и что из регионов было выгружено
for(auto& [worldId, world] : Expanse.Worlds) { for(auto& [worldId, world] : Expanse.Worlds) {
World::SaveUnloadInfo info = world->onStepDatabaseSync(); World::SaveUnloadInfo info = world->onStepDatabaseSync(Content.CM, CurrentTickDuration);
if(!info.ToSave.empty()) { if(!info.ToSave.empty()) {
auto &obj = toDB.ToSave[worldId]; auto &obj = toDB.ToSave[worldId];
@@ -1800,7 +1950,18 @@ IWorldSaveBackend::TickSyncInfo_Out GameServer::stepDatabaseSync() {
} }
// Синхронизируемся с базой // Синхронизируемся с базой
return SaveBackend.World->tickSync(std::move(toDB)); const auto loadFallback = toDB.Load;
try {
return SaveBackend.World->tickSync(std::move(toDB));
} catch(const std::exception& exc) {
LOG.error() << "Ошибка tickSync: " << exc.what();
} catch(...) {
LOG.error() << "Неизвестная ошибка tickSync";
}
IWorldSaveBackend::TickSyncInfo_Out out;
out.NotExisten = loadFallback;
return out;
} }
void GameServer::stepGeneratorAndLuaAsync(IWorldSaveBackend::TickSyncInfo_Out db) { void GameServer::stepGeneratorAndLuaAsync(IWorldSaveBackend::TickSyncInfo_Out db) {
@@ -1840,10 +2001,62 @@ void GameServer::stepGeneratorAndLuaAsync(IWorldSaveBackend::TickSyncInfo_Out db
// Обработка идентификаторов на стороне луа // Обработка идентификаторов на стороне луа
// Трансформация полученных ключей в профили сервера // Трансформация полученных ключей в профили сервера
auto buildRemap = [&](EnumDefContent type, const std::vector<std::string>& idToKey) {
std::vector<ResourceId> remap;
remap.resize(idToKey.size());
for(size_t i = 0; i < idToKey.size(); ++i) {
if(idToKey[i].empty()) {
remap[i] = static_cast<ResourceId>(i);
continue;
}
auto [domain, key] = parseDomainKey(std::string_view(idToKey[i]));
remap[i] = Content.CM.getId(type, domain, key);
}
return remap;
};
auto remapRegion = [&](DB_Region_Out& region) {
std::vector<ResourceId> voxelRemap;
std::vector<ResourceId> nodeRemap;
std::vector<ResourceId> entityRemap;
if(!region.VoxelIdToKey.empty())
voxelRemap = buildRemap(EnumDefContent::Voxel, region.VoxelIdToKey);
if(!region.NodeIdToKey.empty())
nodeRemap = buildRemap(EnumDefContent::Node, region.NodeIdToKey);
if(!region.EntityToKey.empty())
entityRemap = buildRemap(EnumDefContent::Entity, region.EntityToKey);
if(!voxelRemap.empty()) {
for(auto& voxel : region.Voxels) {
if(voxel.VoxelId < voxelRemap.size())
voxel.VoxelId = voxelRemap[voxel.VoxelId];
}
}
if(!nodeRemap.empty()) {
for(auto& chunk : region.Nodes) {
for(auto& node : chunk) {
if(node.NodeId < nodeRemap.size())
node.NodeId = nodeRemap[node.NodeId];
}
}
}
if(!entityRemap.empty()) {
for(auto& entity : region.Entityes) {
auto id = entity.getDefId();
if(id < entityRemap.size())
entity.setDefId(entityRemap[id]);
}
}
};
for(auto& [WorldId_t, regions] : db.LoadedRegions) { for(auto& [WorldId_t, regions] : db.LoadedRegions) {
auto &list = toLoadRegions[WorldId_t]; auto &list = toLoadRegions[WorldId_t];
for(auto& [pos, region] : regions) { for(auto& [pos, region] : regions) {
remapRegion(region);
auto &obj = list.emplace_back(pos, World::RegionIn()).second; auto &obj = list.emplace_back(pos, World::RegionIn()).second;
convertRegionVoxelsToChunks(region.Voxels, obj.Voxels); convertRegionVoxelsToChunks(region.Voxels, obj.Voxels);
obj.Nodes = std::move(region.Nodes); obj.Nodes = std::move(region.Nodes);
@@ -2454,6 +2667,7 @@ void GameServer::stepSyncContent() {
n.NodeId = 4; n.NodeId = 4;
n.Meta = uint8_t((int(nPos.x) + int(nPos.y) + int(nPos.z)) & 0x3); n.Meta = uint8_t((int(nPos.x) + int(nPos.y) + int(nPos.z)) & 0x3);
region->second->IsChunkChanged_Nodes |= 1ull << cPos.pack(); region->second->IsChunkChanged_Nodes |= 1ull << cPos.pack();
region->second->IsChanged = true;
} }
} }
@@ -2471,6 +2685,7 @@ void GameServer::stepSyncContent() {
n.NodeId = 0; n.NodeId = 0;
n.Meta = 0; n.Meta = 0;
region->second->IsChunkChanged_Nodes |= 1ull << cPos.pack(); region->second->IsChunkChanged_Nodes |= 1ull << cPos.pack();
region->second->IsChanged = true;
} }
} }
} }
@@ -2486,10 +2701,13 @@ void GameServer::stepSyncContent() {
std::vector<Net::Packet> packetsToAll; std::vector<Net::Packet> packetsToAll;
{ {
AssetsPreloader::Out_bakeId baked = Content.AM.bakeIdTables(); std::array<
if(!baked.IdToDK.empty()) { std::vector<AssetsManager::BindDomainKeyInfo>,
std::vector<Net::Packet> dkPackets = RemoteClient::makePackets_informateAssets_DK(baked.IdToDK); static_cast<size_t>(EnumAssets::MAX_ENUM)
packetsToAll.insert(packetsToAll.end(), dkPackets.begin(), dkPackets.end()); > baked = Content.AM.bake();
if(hasAnyBindings(baked)) {
packetsToAll.push_back(RemoteClient::makePacket_informateAssets_DK(baked));
} }
} }
@@ -2507,30 +2725,8 @@ void GameServer::stepSyncContent() {
} }
}; };
std::vector<AssetBinaryInfo> binaryResources; std::vector<std::tuple<ResourceFile::Hash_t, std::shared_ptr<const std::u8string>>> binaryResources
for(const Hash_t& hash : full.Hashes) { = Content.AM.getResources(full.Hashes);
std::optional<
std::tuple<EnumAssets, uint32_t, const AssetsPreloader::MediaResource*>
> result = Content.AM.getResource(hash);
if(!result)
continue;
auto& [type, id, media] = *result;
LOG.debug() << "Server sending type=" << assetTypeName(type)
<< " id=" << id
<< " key=" << media->Domain << ':' << media->Key
<< " hash=" << int(media->Hash[0]) << '.'
<< int(media->Hash[1]) << '.'
<< int(media->Hash[2]) << '.'
<< int(media->Hash[3])
<< " size=" << media->Resource->size();
Resource resource(*media->Resource);
binaryResources.push_back(AssetBinaryInfo{
.Data = std::move(resource),
.Hash = media->Hash
});
}
for(std::shared_ptr<RemoteClient>& remoteClient : Game.RemoteClients) { for(std::shared_ptr<RemoteClient>& remoteClient : Game.RemoteClients) {
if(!binaryResources.empty()) if(!binaryResources.empty())

View File

@@ -6,15 +6,14 @@
#include <Common/Net.hpp> #include <Common/Net.hpp>
#include <Common/Lockable.hpp> #include <Common/Lockable.hpp>
#include <atomic> #include <atomic>
#include <barrier>
#include <boost/asio/any_io_executor.hpp> #include <boost/asio/any_io_executor.hpp>
#include <boost/asio/io_context.hpp> #include <boost/asio/io_context.hpp>
#include <condition_variable>
#include <filesystem> #include <filesystem>
#include "Common/Abstract.hpp" #include "Common/Abstract.hpp"
#include "RemoteClient.hpp" #include "RemoteClient.hpp"
#include "Server/Abstract.hpp" #include "Server/Abstract.hpp"
#include <TOSLib.hpp> #include <TOSLib.hpp>
#include <functional>
#include <memory> #include <memory>
#include <queue> #include <queue>
#include <set> #include <set>
@@ -25,6 +24,7 @@
#include "WorldDefManager.hpp" #include "WorldDefManager.hpp"
#include "ContentManager.hpp" #include "ContentManager.hpp"
#include "AssetsManager.hpp"
#include "World.hpp" #include "World.hpp"
#include "SaveBackend.hpp" #include "SaveBackend.hpp"
@@ -73,7 +73,7 @@ class GameServer : public AsyncObject {
struct ContentObj { struct ContentObj {
public: public:
AssetsPreloader AM; AssetsManager AM;
ContentManager CM; ContentManager CM;
// Если контент был перерегистрирован (исключая двоичные ресурсы), то профили будут повторно разосланы // Если контент был перерегистрирован (исключая двоичные ресурсы), то профили будут повторно разосланы
@@ -88,6 +88,8 @@ class GameServer : public AsyncObject {
struct { struct {
std::vector<std::shared_ptr<RemoteClient>> RemoteClients; std::vector<std::shared_ptr<RemoteClient>> RemoteClients;
ServerTime AfterStartTime = {0, 0}; ServerTime AfterStartTime = {0, 0};
// Счётчик тактов (увеличивается на 1 каждый тик в GameServer::run)
uint32_t Tick = 0;
} Game; } Game;
@@ -156,38 +158,56 @@ class GameServer : public AsyncObject {
*/ */
struct BackingChunkPressure_t { struct BackingChunkPressure_t {
TOS::Logger LOG = "BackingChunkPressure"; TOS::Logger LOG = "BackingChunkPressure";
volatile bool NeedShutdown = false; std::atomic<bool> NeedShutdown = false;
std::vector<std::thread> Threads; std::vector<std::thread> Threads;
std::mutex Mutex; std::unique_ptr<std::barrier<>> CollectStart;
volatile int RunCollect = 0, RunCompress = 0, Iteration = 0; std::unique_ptr<std::barrier<>> CollectEnd;
std::condition_variable Symaphore; std::unique_ptr<std::barrier<>> CompressEnd;
std::unordered_map<WorldId_t, std::unique_ptr<World>> *Worlds; std::unordered_map<WorldId_t, std::unique_ptr<World>> *Worlds;
bool HasStarted = false;
void init(size_t threadCount) {
if(threadCount == 0)
return;
const ptrdiff_t participants = static_cast<ptrdiff_t>(threadCount + 1);
CollectStart = std::make_unique<std::barrier<>>(participants);
CollectEnd = std::make_unique<std::barrier<>>(participants);
CompressEnd = std::make_unique<std::barrier<>>(participants);
}
void startCollectChanges() { void startCollectChanges() {
std::lock_guard<std::mutex> lock(Mutex); if(!CollectStart)
RunCollect = Threads.size(); return;
RunCompress = Threads.size(); HasStarted = true;
Iteration += 1; CollectStart->arrive_and_wait();
assert(RunCollect != 0);
Symaphore.notify_all();
} }
void endCollectChanges() { void endCollectChanges() {
std::unique_lock<std::mutex> lock(Mutex); if(!CollectEnd)
Symaphore.wait(lock, [&](){ return RunCollect == 0 || NeedShutdown; }); return;
if(!HasStarted)
return;
CollectEnd->arrive_and_wait();
} }
void endWithResults() { void endWithResults() {
std::unique_lock<std::mutex> lock(Mutex); if(!CompressEnd)
Symaphore.wait(lock, [&](){ return RunCompress == 0 || NeedShutdown; }); return;
if(!HasStarted)
return;
CompressEnd->arrive_and_wait();
} }
void stop() { void stop() {
{ NeedShutdown.store(true, std::memory_order_release);
std::unique_lock<std::mutex> lock(Mutex);
NeedShutdown = true; if(CollectStart)
Symaphore.notify_all(); CollectStart->arrive_and_drop();
} if(CollectEnd)
CollectEnd->arrive_and_drop();
if(CompressEnd)
CompressEnd->arrive_and_drop();
for(std::thread& thread : Threads) for(std::thread& thread : Threads)
thread.join(); thread.join();
@@ -234,7 +254,7 @@ class GameServer : public AsyncObject {
auto lock = Output.lock(); auto lock = Output.lock();
std::vector<std::pair<NoiseKey, std::array<float, 64*64*64>>> out = std::move(*lock); std::vector<std::pair<NoiseKey, std::array<float, 64*64*64>>> out = std::move(*lock);
lock->reserve(8000); lock->reserve(25);
return std::move(out); return std::move(out);
} }
@@ -249,6 +269,13 @@ class GameServer : public AsyncObject {
std::vector<std::thread> Threads; std::vector<std::thread> Threads;
TOS::SpinlockObject<std::queue<std::pair<BackingNoiseGenerator_t::NoiseKey, std::array<float, 64*64*64>>>> NoiseIn; TOS::SpinlockObject<std::queue<std::pair<BackingNoiseGenerator_t::NoiseKey, std::array<float, 64*64*64>>>> NoiseIn;
TOS::SpinlockObject<std::vector<std::pair<BackingNoiseGenerator_t::NoiseKey, World::RegionIn>>> RegionOut; TOS::SpinlockObject<std::vector<std::pair<BackingNoiseGenerator_t::NoiseKey, World::RegionIn>>> RegionOut;
ContentManager &CM;
BackingAsyncLua_t(ContentManager& cm)
: CM(cm)
{
}
void stop() { void stop() {
NeedShutdown = true; NeedShutdown = true;

View File

@@ -11,13 +11,14 @@
#include <boost/system/system_error.hpp> #include <boost/system/system_error.hpp>
#include <exception> #include <exception>
#include <Common/Packets.hpp> #include <Common/Packets.hpp>
#include <unordered_set>
namespace LV::Server { namespace LV::Server {
Net::Packet RemoteClient::makePacket_informateAssets_DK( Net::Packet RemoteClient::makePacket_informateAssets_DK(
const std::array< const std::array<
std::vector<AssetsPreloader::BindDomainKeyInfo>, std::vector<AssetsManager::BindDomainKeyInfo>,
static_cast<size_t>(EnumAssets::MAX_ENUM) static_cast<size_t>(EnumAssets::MAX_ENUM)
>& dkVector >& dkVector
) { ) {
@@ -46,7 +47,7 @@ Net::Packet RemoteClient::makePacket_informateAssets_DK(
// Запись связок домен+ключ // Запись связок домен+ключ
for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); type++) { for(size_t type = 0; type < static_cast<size_t>(EnumAssets::MAX_ENUM); type++) {
const std::vector<AssetsPreloader::BindDomainKeyInfo>& binds = dkVector[type]; const std::vector<AssetsManager::BindDomainKeyInfo>& binds = dkVector[type];
pack << uint32_t(binds.size()); pack << uint32_t(binds.size());
for(const auto& bind : binds) { for(const auto& bind : binds) {
@@ -59,6 +60,7 @@ Net::Packet RemoteClient::makePacket_informateAssets_DK(
// Сжатие // Сжатие
std::u8string compressed = compressLinear(pack.complite()); std::u8string compressed = compressLinear(pack.complite());
pack.clear();
pack << uint8_t(ToClient::AssetsBindDK) << (const std::string&) compressed; pack << uint8_t(ToClient::AssetsBindDK) << (const std::string&) compressed;
return pack; return pack;
@@ -66,7 +68,7 @@ Net::Packet RemoteClient::makePacket_informateAssets_DK(
Net::Packet RemoteClient::makePacket_informateAssets_HH( Net::Packet RemoteClient::makePacket_informateAssets_HH(
const std::array< const std::array<
std::vector<AssetsPreloader::BindHashHeaderInfo>, std::vector<AssetsManager::BindHashHeaderInfo>,
static_cast<size_t>(EnumAssets::MAX_ENUM) static_cast<size_t>(EnumAssets::MAX_ENUM)
>& hhVector, >& hhVector,
const std::array< const std::array<
@@ -92,24 +94,122 @@ Net::Packet RemoteClient::makePacket_informateAssets_HH(
return pack; return pack;
} }
std::vector<Net::Packet> RemoteClient::makePackets_sendDefContentUpdate( std::vector<Net::Packet> RemoteClient::makePackets_informateDefContent_Full(
std::array< const ContentManager::Out_getAllProfiles& profiles
std::vector< ) {
std::pair< std::vector<Net::Packet> packets;
ResourceId, // Идентификатор профиля Net::Packet pack;
std::u8string // Двоичный формат профиля
> auto check = [&](size_t needSize) {
>, if(pack.size()+needSize > 65500) {
static_cast<size_t>(EnumDefContent::MAX_ENUM) packets.emplace_back(std::move(pack));
> newOrUpdate, // Новые или изменённые pack.clear();
std::array< }
std::vector<ResourceId>, };
static_cast<size_t>(EnumDefContent::MAX_ENUM)
> lost, // Потерянные профили pack << (uint8_t) ToClient::DefinitionsFull;
std::array<
std::vector<std::pair<std::string, std::string>>, {
static_cast<size_t>(EnumDefContent::MAX_ENUM) pack << (uint32_t) profiles.ProfilesIds_Voxel.size();
> idToDK // Новые привязки for(uint32_t id : profiles.ProfilesIds_Voxel) {
check(4);
pack << id;
}
for(const auto& profile : profiles.Profiles_Voxel) {
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
}
}
{
pack << (uint32_t) profiles.ProfilesIds_Node.size();
for(uint32_t id : profiles.ProfilesIds_Node) {
check(4);
pack << id;
}
for(const auto& profile : profiles.Profiles_Node) {
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
}
}
{
pack << (uint32_t) profiles.ProfilesIds_World.size();
for(uint32_t id : profiles.ProfilesIds_World) {
check(4);
pack << id;
}
for(const auto& profile : profiles.Profiles_World) {
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
}
}
{
pack << (uint32_t) profiles.ProfilesIds_Portal.size();
for(uint32_t id : profiles.ProfilesIds_Portal) {
check(4);
pack << id;
}
for(const auto& profile : profiles.Profiles_Portal) {
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
}
}
{
pack << (uint32_t) profiles.ProfilesIds_Entity.size();
for(uint32_t id : profiles.ProfilesIds_Entity) {
check(4);
pack << id;
}
for(const auto& profile : profiles.Profiles_Entity) {
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
}
}
{
pack << (uint32_t) profiles.ProfilesIds_Item.size();
for(uint32_t id : profiles.ProfilesIds_Item) {
check(4);
pack << id;
}
for(const auto& profile : profiles.Profiles_Item) {
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
}
}
for(size_t type = 0; type < static_cast<size_t>(EnumDefContent::MAX_ENUM); type++) {
pack << uint32_t(profiles.IdToDK[type].size());
for(const auto& [domain, key] : profiles.IdToDK[type]) {
check(domain.size() + key.size() + 8);
pack << domain << key;
}
}
if(pack.size())
packets.emplace_back(std::move(pack));
return packets;
}
std::vector<Net::Packet> RemoteClient::makePackets_informateDefContentUpdate(
const ContentManager::Out_buildEndProfiles& profiles
) { ) {
std::vector<Net::Packet> packets; std::vector<Net::Packet> packets;
Net::Packet pack; Net::Packet pack;
@@ -122,33 +222,86 @@ std::vector<Net::Packet> RemoteClient::makePackets_sendDefContentUpdate(
}; };
pack << (uint8_t) ToClient::DefinitionsUpdate; pack << (uint8_t) ToClient::DefinitionsUpdate;
pack << uint32_t(newOrUpdate.size());
for(size_t type = 0; type < static_cast<size_t>(EnumDefContent::MAX_ENUM); type++) {
pack << uint32_t(newOrUpdate[type].size());
for(const auto& [id, data] : newOrUpdate[type]) { {
check(data.size()); pack << uint32_t(profiles.ChangedProfiles_Voxel.size());
pack << id << (const std::string&) data; for(const auto& [id, profile] : profiles.ChangedProfiles_Voxel) {
}
}
pack << uint32_t(lost.size());
for(size_t type = 0; type < static_cast<size_t>(EnumDefContent::MAX_ENUM); type++) {
pack << uint32_t(lost[type].size());
for(ResourceId id : lost[type]) {
check(4);
pack << id; pack << id;
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
} }
} }
pack << uint32_t(idToDK.size()); {
for(size_t type = 0; type < static_cast<size_t>(EnumDefContent::MAX_ENUM); type++) { pack << uint32_t(profiles.ChangedProfiles_Node.size());
pack << uint32_t(idToDK[type].size()); for(const auto& [id, profile] : profiles.ChangedProfiles_Node) {
pack << id;
for(const auto& [domain, key] : idToDK[type]) { std::u8string data = profile->dumpToClient();
check(domain.size() + key.size() + 8); check(data.size());
pack << key << domain; pack << std::string_view((const char*) data.data(), data.size());
}
}
{
pack << uint32_t(profiles.ChangedProfiles_World.size());
for(const auto& [id, profile] : profiles.ChangedProfiles_World) {
pack << id;
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
}
}
{
pack << uint32_t(profiles.ChangedProfiles_Portal.size());
for(const auto& [id, profile] : profiles.ChangedProfiles_Portal) {
pack << id;
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
}
}
{
pack << uint32_t(profiles.ChangedProfiles_Entity.size());
for(const auto& [id, profile] : profiles.ChangedProfiles_Entity) {
pack << id;
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
}
}
{
pack << uint32_t(profiles.ChangedProfiles_Item.size());
for(const auto& [id, profile] : profiles.ChangedProfiles_Item) {
pack << id;
std::u8string data = profile->dumpToClient();
check(data.size());
pack << std::string_view((const char*) data.data(), data.size());
}
}
{
for(size_t type = 0; type < static_cast<size_t>(EnumDefContent::MAX_ENUM); type++) {
pack << uint32_t(profiles.LostProfiles[type].size());
for(const ResourceId id : profiles.LostProfiles[type]) {
check(sizeof(ResourceId));
pack << id;
}
}
}
{
for(size_t type = 0; type < static_cast<size_t>(EnumDefContent::MAX_ENUM); type++) {
pack << uint32_t(profiles.IdToDK[type].size());
for(const auto& [domain, key] : profiles.IdToDK[type]) {
check(domain.size() + key.size() + 8);
pack << key << domain;
}
} }
} }
@@ -388,10 +541,8 @@ void RemoteClient::NetworkAndResource_t::prepareWorldRemove(WorldId_t worldId)
void RemoteClient::prepareCameraSetEntity(ServerEntityId_t entityId) { void RemoteClient::prepareCameraSetEntity(ServerEntityId_t entityId) {
auto lock = NetworkAndResource.lock(); auto lock = NetworkAndResource.lock();
ClientEntityId_t cId = lock->ReMapEntities.toClient(entityId); lock->checkPacketBorder(4);
lock->checkPacketBorder(8); lock->NextPacket << (uint8_t) ToClient::TestLinkCameraToEntity;
lock->NextPacket << (uint8_t) ToClient::TestLinkCameraToEntity
<< cId;
} }
ResourceRequest RemoteClient::pushPreparedPackets() { ResourceRequest RemoteClient::pushPreparedPackets() {
@@ -431,21 +582,21 @@ ResourceRequest RemoteClient::pushPreparedPackets() {
return std::move(nextRequest); return std::move(nextRequest);
} }
void RemoteClient::informateBinaryAssets(const std::vector<AssetBinaryInfo>& resources) void RemoteClient::informateBinaryAssets(const std::vector<std::tuple<ResourceFile::Hash_t, std::shared_ptr<const std::u8string>>>& resources)
{ {
for(const AssetBinaryInfo& resource : resources) { for(const auto& [hash, resource] : resources) {
auto lock = NetworkAndResource.lock(); auto lock = NetworkAndResource.lock();
auto iter = std::find(lock->ClientRequested.begin(), lock->ClientRequested.end(), resource.Hash); auto iter = std::find(lock->ClientRequested.begin(), lock->ClientRequested.end(), hash);
if(iter == lock->ClientRequested.end()) if(iter == lock->ClientRequested.end())
continue; continue;
lock->ClientRequested.erase(iter); lock->ClientRequested.erase(iter);
lock.unlock(); lock.unlock();
auto it = std::lower_bound(AssetsInWork.OnClient.begin(), AssetsInWork.OnClient.end(), resource.Hash); auto it = std::lower_bound(AssetsInWork.OnClient.begin(), AssetsInWork.OnClient.end(), hash);
if(it == AssetsInWork.OnClient.end() || *it != resource.Hash) { if(it == AssetsInWork.OnClient.end() || *it != hash) {
AssetsInWork.OnClient.insert(it, resource.Hash); AssetsInWork.OnClient.insert(it, hash);
AssetsInWork.ToSend.emplace_back(resource.Data, 0); AssetsInWork.ToSend.emplace_back(hash, resource, 0);
} else { } else {
LOG.warn() << "Клиент повторно запросил имеющийся у него ресурс"; LOG.warn() << "Клиент повторно запросил имеющийся у него ресурс";
} }
@@ -612,36 +763,36 @@ void RemoteClient::onUpdate() {
bool hasFullSended = false; bool hasFullSended = false;
for(auto& [res, sended] : toSend) { for(auto& [hash, res, sended] : toSend) {
if(sended == 0) { if(sended == 0) {
// Оповещаем о начале отправки ресурса // Оповещаем о начале отправки ресурса
const size_t initSize = 1 + 1 + 4 + 32 + 4 + 1; const size_t initSize = 1 + 1 + 4 + 32 + 4 + 1;
if(p.size() + initSize > kMaxAssetPacketSize) if(p.size() + initSize > kMaxAssetPacketSize)
flushAssetsPacket(); flushAssetsPacket();
p << (uint8_t) ToClient::AssetsInitSend p << (uint8_t) ToClient::AssetsInitSend
<< uint32_t(res.size()); << uint32_t(res->size());
p.write((const std::byte*) res.hash().data(), 32); p.write((const std::byte*) hash.data(), 32);
} }
// Отправляем чанк // Отправляем чанк
size_t willSend = std::min(chunkSize, res.size()-sended); size_t willSend = std::min(chunkSize, res->size()-sended);
const size_t chunkMsgSize = 1 + 1 + 32 + 4 + willSend; const size_t chunkMsgSize = 1 + 1 + 32 + 4 + willSend;
if(p.size() + chunkMsgSize > kMaxAssetPacketSize) if(p.size() + chunkMsgSize > kMaxAssetPacketSize)
flushAssetsPacket(); flushAssetsPacket();
p << (uint8_t) ToClient::AssetsNextSend; p << (uint8_t) ToClient::AssetsNextSend;
p.write((const std::byte*) res.hash().data(), 32); p.write((const std::byte*) hash.data(), 32);
p << uint32_t(willSend); p << uint32_t(willSend);
p.write(res.data() + sended, willSend); p.write((const std::byte*) res->data() + sended, willSend);
sended += willSend; sended += willSend;
if(sended == res.size()) { if(sended == res->size()) {
hasFullSended = true; hasFullSended = true;
} }
} }
if(hasFullSended) { if(hasFullSended) {
for(ssize_t iter = toSend.size()-1; iter >= 0; iter--) { for(ssize_t iter = toSend.size()-1; iter >= 0; iter--) {
if(std::get<0>(toSend[iter]).size() == std::get<1>(toSend[iter])) { if(std::get<1>(toSend[iter])->size() == std::get<2>(toSend[iter])) {
toSend.erase(toSend.begin()+iter); toSend.erase(toSend.begin()+iter);
} }
} }

View File

@@ -5,6 +5,7 @@
#include <Common/Net.hpp> #include <Common/Net.hpp>
#include "Abstract.hpp" #include "Abstract.hpp"
#include "Common/Packets.hpp" #include "Common/Packets.hpp"
#include "Server/AssetsManager.hpp"
#include "Server/ContentManager.hpp" #include "Server/ContentManager.hpp"
#include <Common/Abstract.hpp> #include <Common/Abstract.hpp>
#include <bitset> #include <bitset>
@@ -256,7 +257,7 @@ class RemoteClient {
std::vector<Hash_t> OnClient; std::vector<Hash_t> OnClient;
// Отправляемые на клиент ресурсы // Отправляемые на клиент ресурсы
// Ресурс, количество отправленных байт // Ресурс, количество отправленных байт
std::vector<std::tuple<Resource, size_t>> ToSend; std::vector<std::tuple<ResourceFile::Hash_t, std::shared_ptr<const std::u8string>, size_t>> ToSend;
// Пакет с ресурсами // Пакет с ресурсами
std::vector<Net::Packet> AssetsPackets; std::vector<Net::Packet> AssetsPackets;
Net::Packet AssetsPacket; Net::Packet AssetsPacket;
@@ -276,6 +277,10 @@ public:
ContentViewInfo ContentViewState; ContentViewInfo ContentViewState;
// Если игрок пересекал границы региона (для перерасчёта ContentViewState) // Если игрок пересекал границы региона (для перерасчёта ContentViewState)
bool CrossedRegion = true; bool CrossedRegion = true;
// Отложенная выгрузка регионов (гистерезис + задержка)
// worldId -> (regionPos -> tick_deadline)
std::unordered_map<WorldId_t, std::unordered_map<Pos::GlobalRegion, uint32_t>> PendingRegionUnload;
std::queue<Pos::GlobalNode> Build, Break; std::queue<Pos::GlobalNode> Build, Break;
std::optional<ServerEntityId_t> PlayerEntity; std::optional<ServerEntityId_t> PlayerEntity;
@@ -361,7 +366,7 @@ public:
// Создаёт пакет для всех игроков с оповещением о новых идентификаторах (id -> domain+key) // Создаёт пакет для всех игроков с оповещением о новых идентификаторах (id -> domain+key)
static Net::Packet makePacket_informateAssets_DK( static Net::Packet makePacket_informateAssets_DK(
const std::array< const std::array<
std::vector<AssetsPreloader::BindDomainKeyInfo>, std::vector<AssetsManager::BindDomainKeyInfo>,
static_cast<size_t>(EnumAssets::MAX_ENUM) static_cast<size_t>(EnumAssets::MAX_ENUM)
>& dkVector >& dkVector
); );
@@ -369,7 +374,7 @@ public:
// Создаёт пакет для всех игроков с оповещением об изменении файлов ресурсов (id -> hash+header) // Создаёт пакет для всех игроков с оповещением об изменении файлов ресурсов (id -> hash+header)
static Net::Packet makePacket_informateAssets_HH( static Net::Packet makePacket_informateAssets_HH(
const std::array< const std::array<
std::vector<AssetsPreloader::BindHashHeaderInfo>, std::vector<AssetsManager::BindHashHeaderInfo>,
static_cast<size_t>(EnumAssets::MAX_ENUM) static_cast<size_t>(EnumAssets::MAX_ENUM)
>& hhVector, >& hhVector,
const std::array< const std::array<
@@ -380,28 +385,18 @@ public:
// Оповещение о двоичных ресурсах (стриминг по запросу) // Оповещение о двоичных ресурсах (стриминг по запросу)
void informateBinaryAssets( void informateBinaryAssets(
const std::vector<AssetBinaryInfo>& resources const std::vector<std::tuple<ResourceFile::Hash_t, std::shared_ptr<const std::u8string>>>& resources
); );
// Создаёт пакет со всеми данными об игровых профилях
static std::vector<Net::Packet> makePackets_informateDefContent_Full(
const ContentManager::Out_getAllProfiles& profiles
);
// Создаёт пакет об обновлении игровых профилей // Создаёт пакет об обновлении игровых профилей
static std::vector<Net::Packet> makePackets_sendDefContentUpdate( static std::vector<Net::Packet> makePackets_informateDefContentUpdate(
std::array< const ContentManager::Out_buildEndProfiles& profiles
std::vector<
std::pair<
ResourceId, // Идентификатор профиля
std::u8string // Двоичный формат профиля
>
>,
static_cast<size_t>(EnumDefContent::MAX_ENUM)
> newOrUpdate, // Новые или изменённые
std::array<
std::vector<ResourceId>,
static_cast<size_t>(EnumDefContent::MAX_ENUM)
> lost, // Потерянные профили
std::array<
std::vector<std::pair<std::string, std::string>>,
static_cast<size_t>(EnumDefContent::MAX_ENUM)
> idToDK // Новые привязки
); );
void onUpdate(); void onUpdate();
@@ -411,13 +406,6 @@ private:
void protocolError(); void protocolError();
coro<> readPacket(Net::AsyncSocket &sock); coro<> readPacket(Net::AsyncSocket &sock);
coro<> rP_System(Net::AsyncSocket &sock); coro<> rP_System(Net::AsyncSocket &sock);
// void incrementProfile(const std::vector<TextureId_t> &textures, const std::vector<ModelId_t> &model,
// const std::vector<SoundId_t> &sounds, const std::vector<FontId_t> &font
// );
// void decrementProfile(std::vector<TextureId_t> &&textures, std::vector<ModelId_t> &&model,
// std::vector<SoundId_t> &&sounds, std::vector<FontId_t> &&font
// );
}; };

View File

@@ -16,7 +16,7 @@ namespace LV::Server {
*/ */
struct SB_Region_In { struct SB_Region_In {
// Список вокселей всех чанков // Список вокселей всех чанков
std::unordered_map<Pos::bvec4u, VoxelCube> Voxels; std::unordered_map<Pos::bvec4u, std::vector<VoxelCube>> Voxels;
// Привязка вокселей к ключу профиля // Привязка вокселей к ключу профиля
std::vector<std::pair<DefVoxelId, std::string>> VoxelsMap; std::vector<std::pair<DefVoxelId, std::string>> VoxelsMap;
// Ноды всех чанков // Ноды всех чанков
@@ -132,4 +132,4 @@ public:
}; };
} }

View File

@@ -1,6 +1,7 @@
#include "Filesystem.hpp" #include "Filesystem.hpp"
#include "Server/Abstract.hpp" #include "Server/Abstract.hpp"
#include "Server/SaveBackend.hpp" #include "Server/SaveBackend.hpp"
#include "TOSLib.hpp"
#include <boost/json/array.hpp> #include <boost/json/array.hpp>
#include <boost/json/object.hpp> #include <boost/json/object.hpp>
#include <boost/json/parse.hpp> #include <boost/json/parse.hpp>
@@ -11,12 +12,277 @@
#include <filesystem> #include <filesystem>
#include <fstream> #include <fstream>
#include <string> #include <string>
#include <vector>
#include <algorithm>
#include <cstring>
namespace LV::Server::SaveBackends { namespace LV::Server::SaveBackends {
namespace fs = std::filesystem; namespace fs = std::filesystem;
namespace js = boost::json; namespace js = boost::json;
namespace {
constexpr uint32_t kRegionVersion = 1;
constexpr size_t kRegionNodeCount = 4 * 4 * 4 * 16 * 16 * 16;
template<typename T>
js::object packIdMap(const std::vector<std::pair<T, std::string>>& map) {
js::object out;
for(const auto& [id, key] : map) {
out[std::to_string(id)] = key;
}
return out;
}
void unpackIdMap(const js::object& obj, std::vector<std::string>& out) {
size_t maxId = 0;
for(const auto& kvp : obj) {
try {
maxId = std::max(maxId, static_cast<size_t>(std::stoul(kvp.key())));
} catch(...) {
continue;
}
}
out.assign(maxId + 1, {});
for(const auto& kvp : obj) {
try {
size_t id = std::stoul(kvp.key());
out[id] = std::string(kvp.value().as_string());
} catch(...) {
continue;
}
}
}
std::string encodeCompressed(const uint8_t* data, size_t size) {
std::u8string compressed = compressLinear(std::u8string_view(reinterpret_cast<const char8_t*>(data), size));
return TOS::Enc::toBase64(reinterpret_cast<const uint8_t*>(compressed.data()), compressed.size());
}
std::u8string decodeCompressed(const std::string& base64) {
if(base64.empty())
return {};
TOS::ByteBuffer buffer = TOS::Enc::fromBase64(base64);
return unCompressLinear(std::u8string_view(reinterpret_cast<const char8_t*>(buffer.data()), buffer.size()));
}
bool writeRegionFile(const fs::path& path, const SB_Region_In& data) {
js::object jobj;
jobj["version"] = kRegionVersion;
{
std::vector<VoxelCube_Region> voxels;
convertChunkVoxelsToRegion(data.Voxels, voxels);
js::object jvoxels;
jvoxels["count"] = static_cast<uint64_t>(voxels.size());
if(!voxels.empty()) {
const uint8_t* raw = reinterpret_cast<const uint8_t*>(voxels.data());
size_t rawSize = sizeof(VoxelCube_Region) * voxels.size();
jvoxels["data"] = encodeCompressed(raw, rawSize);
} else {
jvoxels["data"] = "";
}
jobj["voxels"] = std::move(jvoxels);
jobj["voxels_map"] = packIdMap(data.VoxelsMap);
}
{
js::object jnodes;
const Node* nodePtr = data.Nodes[0].data();
const uint8_t* raw = reinterpret_cast<const uint8_t*>(nodePtr);
size_t rawSize = sizeof(Node) * kRegionNodeCount;
jnodes["data"] = encodeCompressed(raw, rawSize);
jobj["nodes"] = std::move(jnodes);
jobj["nodes_map"] = packIdMap(data.NodeMap);
}
{
js::array ents;
for(const Entity& entity : data.Entityes) {
js::object je;
je["def"] = static_cast<uint64_t>(entity.getDefId());
je["world"] = static_cast<uint64_t>(entity.WorldId);
je["pos"] = js::array{entity.Pos.x, entity.Pos.y, entity.Pos.z};
je["speed"] = js::array{entity.Speed.x, entity.Speed.y, entity.Speed.z};
je["accel"] = js::array{entity.Acceleration.x, entity.Acceleration.y, entity.Acceleration.z};
je["quat"] = js::array{entity.Quat.x, entity.Quat.y, entity.Quat.z, entity.Quat.w};
je["hp"] = static_cast<uint64_t>(entity.HP);
je["abbox"] = js::array{entity.ABBOX.x, entity.ABBOX.y, entity.ABBOX.z};
je["in_region"] = js::array{entity.InRegionPos.x, entity.InRegionPos.y, entity.InRegionPos.z};
js::object tags;
for(const auto& [key, value] : entity.Tags) {
tags[key] = value;
}
je["tags"] = std::move(tags);
ents.push_back(std::move(je));
}
jobj["entities"] = std::move(ents);
jobj["entities_map"] = packIdMap(data.EntityMap);
}
fs::create_directories(path.parent_path());
std::ofstream fd(path, std::ios::binary);
if(!fd)
return false;
fd << js::serialize(jobj);
return true;
}
bool readRegionFile(const fs::path& path, DB_Region_Out& out) {
try {
std::ifstream fd(path, std::ios::binary);
if(!fd)
return false;
out = {};
js::object jobj = js::parse(fd).as_object();
if(auto it = jobj.find("voxels"); it != jobj.end()) {
const js::object& jvoxels = it->value().as_object();
size_t count = 0;
if(auto itCount = jvoxels.find("count"); itCount != jvoxels.end())
count = static_cast<size_t>(itCount->value().to_number<uint64_t>());
std::string base64;
if(auto itData = jvoxels.find("data"); itData != jvoxels.end())
base64 = std::string(itData->value().as_string());
if(count > 0 && !base64.empty()) {
std::u8string raw = decodeCompressed(base64);
if(raw.size() != sizeof(VoxelCube_Region) * count)
return false;
out.Voxels.resize(count);
std::memcpy(out.Voxels.data(), raw.data(), raw.size());
}
}
if(auto it = jobj.find("voxels_map"); it != jobj.end()) {
unpackIdMap(it->value().as_object(), out.VoxelIdToKey);
}
if(auto it = jobj.find("nodes"); it != jobj.end()) {
const js::object& jnodes = it->value().as_object();
std::string base64;
if(auto itData = jnodes.find("data"); itData != jnodes.end())
base64 = std::string(itData->value().as_string());
if(!base64.empty()) {
std::u8string raw = decodeCompressed(base64);
if(raw.size() != sizeof(Node) * kRegionNodeCount)
return false;
std::memcpy(out.Nodes[0].data(), raw.data(), raw.size());
}
}
if(auto it = jobj.find("nodes_map"); it != jobj.end()) {
unpackIdMap(it->value().as_object(), out.NodeIdToKey);
}
if(auto it = jobj.find("entities"); it != jobj.end()) {
const js::array& ents = it->value().as_array();
out.Entityes.reserve(ents.size());
for(const js::value& val : ents) {
const js::object& je = val.as_object();
DefEntityId defId = static_cast<DefEntityId>(je.at("def").to_number<uint64_t>());
Entity entity(defId);
if(auto itWorld = je.find("world"); itWorld != je.end())
entity.WorldId = static_cast<DefWorldId>(itWorld->value().to_number<uint64_t>());
if(auto itPos = je.find("pos"); itPos != je.end()) {
const js::array& arr = itPos->value().as_array();
entity.Pos = Pos::Object(
static_cast<int32_t>(arr.at(0).to_number<int64_t>()),
static_cast<int32_t>(arr.at(1).to_number<int64_t>()),
static_cast<int32_t>(arr.at(2).to_number<int64_t>())
);
}
if(auto itSpeed = je.find("speed"); itSpeed != je.end()) {
const js::array& arr = itSpeed->value().as_array();
entity.Speed = Pos::Object(
static_cast<int32_t>(arr.at(0).to_number<int64_t>()),
static_cast<int32_t>(arr.at(1).to_number<int64_t>()),
static_cast<int32_t>(arr.at(2).to_number<int64_t>())
);
}
if(auto itAccel = je.find("accel"); itAccel != je.end()) {
const js::array& arr = itAccel->value().as_array();
entity.Acceleration = Pos::Object(
static_cast<int32_t>(arr.at(0).to_number<int64_t>()),
static_cast<int32_t>(arr.at(1).to_number<int64_t>()),
static_cast<int32_t>(arr.at(2).to_number<int64_t>())
);
}
if(auto itQuat = je.find("quat"); itQuat != je.end()) {
const js::array& arr = itQuat->value().as_array();
entity.Quat = glm::quat(
static_cast<float>(arr.at(3).to_number<double>()),
static_cast<float>(arr.at(0).to_number<double>()),
static_cast<float>(arr.at(1).to_number<double>()),
static_cast<float>(arr.at(2).to_number<double>())
);
}
if(auto itHp = je.find("hp"); itHp != je.end())
entity.HP = static_cast<uint32_t>(itHp->value().to_number<uint64_t>());
if(auto itAabb = je.find("abbox"); itAabb != je.end()) {
const js::array& arr = itAabb->value().as_array();
entity.ABBOX.x = static_cast<uint64_t>(arr.at(0).to_number<uint64_t>());
entity.ABBOX.y = static_cast<uint64_t>(arr.at(1).to_number<uint64_t>());
entity.ABBOX.z = static_cast<uint64_t>(arr.at(2).to_number<uint64_t>());
}
if(auto itRegion = je.find("in_region"); itRegion != je.end()) {
const js::array& arr = itRegion->value().as_array();
entity.InRegionPos = Pos::GlobalRegion(
static_cast<int16_t>(arr.at(0).to_number<int64_t>()),
static_cast<int16_t>(arr.at(1).to_number<int64_t>()),
static_cast<int16_t>(arr.at(2).to_number<int64_t>())
);
}
if(auto itTags = je.find("tags"); itTags != je.end()) {
const js::object& tags = itTags->value().as_object();
for(const auto& kvp : tags) {
entity.Tags[std::string(kvp.key())] = static_cast<float>(kvp.value().to_number<double>());
}
}
out.Entityes.push_back(std::move(entity));
}
}
if(auto it = jobj.find("entities_map"); it != jobj.end()) {
unpackIdMap(it->value().as_object(), out.EntityToKey);
}
return true;
} catch(const std::exception& exc) {
TOS::Logger("RegionLoader::Filesystem").warn() << "Не удалось загрузить регион " << path << "\n\t" << exc.what();
return false;
}
}
}
class WSB_Filesystem : public IWorldSaveBackend { class WSB_Filesystem : public IWorldSaveBackend {
fs::path Dir; fs::path Dir;
@@ -35,7 +301,32 @@ public:
virtual TickSyncInfo_Out tickSync(TickSyncInfo_In &&data) override { virtual TickSyncInfo_Out tickSync(TickSyncInfo_In &&data) override {
TickSyncInfo_Out out; TickSyncInfo_Out out;
out.NotExisten = std::move(data.Load); // Сохранение регионов
for(auto& [worldId, regions] : data.ToSave) {
for(auto& [regionPos, region] : regions) {
writeRegionFile(getPath(std::to_string(worldId), regionPos), region);
}
}
// Загрузка регионов
for(auto& [worldId, regions] : data.Load) {
for(const Pos::GlobalRegion& regionPos : regions) {
const fs::path path = getPath(std::to_string(worldId), regionPos);
if(!fs::exists(path)) {
out.NotExisten[worldId].push_back(regionPos);
continue;
}
DB_Region_Out regionOut;
if(!readRegionFile(path, regionOut)) {
out.NotExisten[worldId].push_back(regionPos);
continue;
}
out.LoadedRegions[worldId].push_back({regionPos, std::move(regionOut)});
}
}
return out; return out;
} }

View File

@@ -1,6 +1,8 @@
#include "World.hpp" #include "World.hpp"
#include "ContentManager.hpp"
#include "TOSLib.hpp" #include "TOSLib.hpp"
#include <memory> #include <memory>
#include <unordered_set>
namespace LV::Server { namespace LV::Server {
@@ -96,8 +98,94 @@ void World::onRemoteClient_RegionsLost(WorldId_t worldId, std::shared_ptr<Remote
} }
} }
World::SaveUnloadInfo World::onStepDatabaseSync() { World::SaveUnloadInfo World::onStepDatabaseSync(ContentManager& cm, float dtime) {
return {}; SaveUnloadInfo out;
constexpr float kSaveDelay = 15.0f;
constexpr float kUnloadDelay = 15.0f;
std::vector<Pos::GlobalRegion> toErase;
toErase.reserve(16);
for(auto& [pos, regionPtr] : Regions) {
Region& region = *regionPtr;
region.LastSaveTime += dtime;
const bool hasChanges = region.IsChanged || region.IsChunkChanged_Voxels || region.IsChunkChanged_Nodes;
const bool needToSave = hasChanges && region.LastSaveTime > kSaveDelay;
const bool needToUnload = region.RMs.empty() && region.LastSaveTime > kUnloadDelay;
if(needToSave || needToUnload) {
SB_Region_In data;
data.Voxels = region.Voxels;
data.Nodes = region.Nodes;
data.Entityes.reserve(region.Entityes.size());
for(const Entity& entity : region.Entityes) {
if(entity.IsRemoved || entity.NeedRemove)
continue;
data.Entityes.push_back(entity);
}
std::unordered_set<DefVoxelId> voxelIds;
for(const auto& [chunkPos, voxels] : region.Voxels) {
(void) chunkPos;
for(const VoxelCube& cube : voxels)
voxelIds.insert(cube.VoxelId);
}
std::unordered_set<DefNodeId> nodeIds;
for(const auto& chunk : region.Nodes) {
for(const Node& node : chunk)
nodeIds.insert(node.NodeId);
}
std::unordered_set<DefEntityId> entityIds;
for(const Entity& entity : data.Entityes)
entityIds.insert(entity.getDefId());
data.VoxelsMap.reserve(voxelIds.size());
for(DefVoxelId id : voxelIds) {
auto dk = cm.getDK(EnumDefContent::Voxel, id);
if(!dk)
continue;
data.VoxelsMap.emplace_back(id, dk->Domain + ":" + dk->Key);
}
data.NodeMap.reserve(nodeIds.size());
for(DefNodeId id : nodeIds) {
auto dk = cm.getDK(EnumDefContent::Node, id);
if(!dk)
continue;
data.NodeMap.emplace_back(id, dk->Domain + ":" + dk->Key);
}
data.EntityMap.reserve(entityIds.size());
for(DefEntityId id : entityIds) {
auto dk = cm.getDK(EnumDefContent::Entity, id);
if(!dk)
continue;
data.EntityMap.emplace_back(id, dk->Domain + ":" + dk->Key);
}
out.ToSave.push_back({pos, std::move(data)});
region.LastSaveTime = 0.0f;
region.IsChanged = false;
}
if(needToUnload) {
out.ToUnload.push_back(pos);
toErase.push_back(pos);
}
}
for(const Pos::GlobalRegion& pos : toErase) {
Regions.erase(pos);
}
return out;
} }
void World::pushRegions(std::vector<std::pair<Pos::GlobalRegion, RegionIn>> regions) { void World::pushRegions(std::vector<std::pair<Pos::GlobalRegion, RegionIn>> regions) {

View File

@@ -12,6 +12,7 @@
namespace LV::Server { namespace LV::Server {
class GameServer; class GameServer;
class ContentManager;
class Region { class Region {
public: public:
@@ -152,7 +153,7 @@ public:
std::vector<Pos::GlobalRegion> ToUnload; std::vector<Pos::GlobalRegion> ToUnload;
std::vector<std::pair<Pos::GlobalRegion, SB_Region_In>> ToSave; std::vector<std::pair<Pos::GlobalRegion, SB_Region_In>> ToSave;
}; };
SaveUnloadInfo onStepDatabaseSync(); SaveUnloadInfo onStepDatabaseSync(ContentManager& cm, float dtime);
struct RegionIn { struct RegionIn {
std::unordered_map<Pos::bvec4u, std::vector<VoxelCube>> Voxels; std::unordered_map<Pos::bvec4u, std::vector<VoxelCube>> Voxels;

View File

@@ -338,9 +338,10 @@ class ByteBuffer : public std::vector<uint8_t> {
if(Index + sizeof(T) > Obj->size()) if(Index + sizeof(T) > Obj->size())
throw std::runtime_error("Вышли за пределы буфера"); throw std::runtime_error("Вышли за пределы буфера");
const uint8_t *ptr = Obj->data()+Index; T value{};
Index += sizeof(T); std::memcpy(&value, Obj->data() + Index, sizeof(T));
return swapEndian(*(const T*) ptr); Index += sizeof(T);
return swapEndian(value);
} }
public: public:
@@ -362,8 +363,8 @@ class ByteBuffer : public std::vector<uint8_t> {
inline Reader& operator>>(int64_t &value) { value = readOffset<int64_t>(); return *this; } inline Reader& operator>>(int64_t &value) { value = readOffset<int64_t>(); return *this; }
inline Reader& operator>>(uint64_t &value) { value = readOffset<uint64_t>(); return *this; } inline Reader& operator>>(uint64_t &value) { value = readOffset<uint64_t>(); return *this; }
inline Reader& operator>>(bool &value) { value = readOffset<uint8_t>(); return *this; } inline Reader& operator>>(bool &value) { value = readOffset<uint8_t>(); return *this; }
inline Reader& operator>>(float &value) { return operator>>(*(uint32_t*) &value); } inline Reader& operator>>(float &value) { uint32_t raw = readOffset<uint32_t>(); std::memcpy(&value, &raw, sizeof(raw)); return *this; }
inline Reader& operator>>(double &value) { return operator>>(*(uint64_t*) &value); } inline Reader& operator>>(double &value) { uint64_t raw = readOffset<uint64_t>(); std::memcpy(&value, &raw, sizeof(raw)); return *this; }
inline int8_t readInt8() { int8_t value; this->operator>>(value); return value; } inline int8_t readInt8() { int8_t value; this->operator>>(value); return value; }
inline uint8_t readUInt8() { uint8_t value; this->operator>>(value); return value; } inline uint8_t readUInt8() { uint8_t value; this->operator>>(value); return value; }
@@ -449,6 +450,17 @@ class ByteBuffer : public std::vector<uint8_t> {
size_t Index = 0; size_t Index = 0;
uint16_t BlockSize = 256; uint16_t BlockSize = 256;
template<typename T> inline void writeRaw(const T &value)
{
uint8_t *ptr = checkBorder(sizeof(T));
std::memcpy(ptr, &value, sizeof(T));
}
template<typename T> inline void writeSwapped(const T &value)
{
T temp = swapEndian(value);
writeRaw(temp);
}
inline uint8_t* checkBorder(size_t count) inline uint8_t* checkBorder(size_t count)
{ {
@@ -469,17 +481,17 @@ class ByteBuffer : public std::vector<uint8_t> {
Writer& operator=(const Writer&) = default; Writer& operator=(const Writer&) = default;
Writer& operator=(Writer&&) = default; Writer& operator=(Writer&&) = default;
inline Writer& operator<<(const int8_t &value) { *(int8_t*) checkBorder(sizeof(value)) = value; return *this; } inline Writer& operator<<(const int8_t &value) { writeRaw(value); return *this; }
inline Writer& operator<<(const uint8_t &value) { *(uint8_t*) checkBorder(sizeof(value)) = value; return *this; } inline Writer& operator<<(const uint8_t &value) { writeRaw(value); return *this; }
inline Writer& operator<<(const int16_t &value) { *(int16_t*) checkBorder(sizeof(value)) = swapEndian(value); return *this; } inline Writer& operator<<(const int16_t &value) { writeSwapped(value); return *this; }
inline Writer& operator<<(const uint16_t &value) { *(uint16_t*) checkBorder(sizeof(value)) = swapEndian(value); return *this; } inline Writer& operator<<(const uint16_t &value) { writeSwapped(value); return *this; }
inline Writer& operator<<(const int32_t &value) { *(int32_t*) checkBorder(sizeof(value)) = swapEndian(value); return *this; } inline Writer& operator<<(const int32_t &value) { writeSwapped(value); return *this; }
inline Writer& operator<<(const uint32_t &value) { *(uint32_t*) checkBorder(sizeof(value)) = swapEndian(value); return *this; } inline Writer& operator<<(const uint32_t &value) { writeSwapped(value); return *this; }
inline Writer& operator<<(const int64_t &value) { *(int64_t*) checkBorder(sizeof(value)) = swapEndian(value); return *this; } inline Writer& operator<<(const int64_t &value) { writeSwapped(value); return *this; }
inline Writer& operator<<(const uint64_t &value) { *(uint64_t*) checkBorder(sizeof(value)) = swapEndian(value); return *this; } inline Writer& operator<<(const uint64_t &value) { writeSwapped(value); return *this; }
inline Writer& operator<<(const bool &value) { *(uint8_t*) checkBorder(sizeof(value)) = uint8_t(value ? 1 : 0); return *this; } inline Writer& operator<<(const bool &value) { uint8_t temp = value ? 1 : 0; writeRaw(temp); return *this; }
inline Writer& operator<<(const float &value) { *(uint32_t*) checkBorder(sizeof(value)) = swapEndian(*(uint32_t*) &value); return *this; } inline Writer& operator<<(const float &value) { uint32_t raw; std::memcpy(&raw, &value, sizeof(raw)); writeSwapped(raw); return *this; }
inline Writer& operator<<(const double &value) { *(uint64_t*) checkBorder(sizeof(value)) = swapEndian(*(uint64_t*) &value); return *this; } inline Writer& operator<<(const double &value) { uint64_t raw; std::memcpy(&raw, &value, sizeof(raw)); writeSwapped(raw); return *this; }
inline void writeInt8(const int8_t &value) { this->operator<<(value); } inline void writeInt8(const int8_t &value) { this->operator<<(value); }
inline void writeUInt8(const uint8_t &value) { this->operator<<(value); } inline void writeUInt8(const uint8_t &value) { this->operator<<(value); }

66
docs/assets_manager.md Normal file
View File

@@ -0,0 +1,66 @@
# AssetsManager
Документ описывает реализацию `AssetsManager` на стороне клиента.
## Назначение
`AssetsManager` объединяет в одном объекте:
- таблицы привязок ресурсов (domain/key -> localId, serverId -> localId);
- загрузку и хранение ресурсов из ресурспаков;
- систему источников данных (packs/memory/cache) с асинхронной выдачей;
- перестройку заголовков ресурсов под локальные идентификаторы.
## Основные структуры данных
- `PerType` — набор таблиц на каждый тип ресурса:
- `DKToLocal` и `LocalToDK` — двунаправленное отображение domain/key <-> localId.
- `LocalParent` — union-find для ребиндинга локальных id.
- `ServerToLocal` и `BindInfos` — привязки серверных id и их метаданные.
- `PackResources` — набор ресурсов, собранных из паков.
- `Sources` — список источников ресурсов (pack, memory, cache).
- `SourceCacheByHash` — кэш успешного источника для хеша.
- `PendingReadsByHash` и `ReadyReads` — очередь ожидания и готовые ответы.
## Источники ресурсов
Источники реализованы через интерфейс `IResourceSource`:
- pack source (sync) — ищет ресурсы в `PackResources`.
- memory source (sync) — ищет в `MemoryResourcesByHash`.
- cache source (async) — делает чтения через `AssetsCacheManager`.
Алгоритм поиска:
1) Сначала проверяется `SourceCacheByHash` (если не протух по поколению).
2) Источники опрашиваются по порядку, первый `Hit` возвращается сразу.
3) Если источник вернул `Pending`, запрос попадает в ожидание.
4) `tickSources()` опрашивает асинхронные источники и переводит ответы в `ReadyReads`.
## Привязка идентификаторов
- `getOrCreateLocalId()` создаёт локальный id для domain/key.
- `bindServerResource()` связывает serverId с localId и записывает `BindInfo`.
- `unionLocalIds()` объединяет локальные id при конфликте, используя union-find.
## Ресурспаки
`reloadPacks()` сканирует директории, собирает ресурсы в `PackResources`,
а затем возвращает список изменений и потерь по типам.
Важно: ключи ресурсов всегда хранятся с разделителем `/`.
Для нормализации пути используется `fs::path::generic_string()`.
## Заголовки
- `rebindHeader()` заменяет id зависимостей в заголовках ресурса.
- `parseHeader()` парсит заголовок без модификаций.
## Поток данных чтения
1) `pushReads()` принимает список `ResourceKey` и пытается получить ресурс.
2) `pullReads()` возвращает готовые ответы, включая промахи.
3) `pushResources()` добавляет ресурсы в память и прокидывает их в кэш.
## Ограничения
- Класс не предназначен для внешнего многопоточного использования.
- Политика приоритета ресурсов в паке фиксированная: первый найденный ключ побеждает.
- Коллизии хешей не обрабатываются отдельно.

105
docs/texture_pipeline.md Normal file
View File

@@ -0,0 +1,105 @@
# Текстурные программы (TexturePipelineProgram)
Текстурная программа — это строка, описывающая источник текстуры и цепочку операций
над ней. Такие строки используются в `textures` моделей и компилируются
`TexturePipelineProgram`.
## Общая форма
```
[tex] <base> [|> op(...)]*
```
`tex` в начале необязателен.
## Базовые выражения
- `name` или `"name.png"` — ссылка на текстуру из assets. Расширение .png/.jpg/.jpeg допустимо.
- `anim(...)` — анимация из спрайт-листа (см. ниже).
- `<w>x<h> <#RRGGBB|#RRGGBBAA>` — заливка цветом.
Примеры:
```
stone
tex "core:stone.png"
32x32 "#FF00FF"
```
## Аргументы операций
- Позиционные: `op(1, 2, "str")`
- Именованные: `op(w=16, h=16)`
- Значения: числа (uint32), строки в кавычках, либо идентификаторы.
Цвета задаются `#RRGGBB` или `#RRGGBBAA`.
## Операции пайплайна
Операции без аргументов можно писать без `()`: `brighten` и т.п.
В подвыражениях текстур (см. ниже) операции без аргументов нужно писать со скобками:
`brighten()`.
### Операции, принимающие текстуру
- `overlay(tex)` — наложение с альфой.
- `mask(tex)` — применение альфа-маски.
- `lowpart(percent, tex)` — смешивание нижней части (percent 1..100).
`tex` может быть:
- именем текстуры: `overlay("core:stone")`
- именованным аргументом: `overlay(tex="core:stone")`
- вложенной программой: `overlay( tex stone |> invert("rgb") )`
### Геометрия и альфа
- `resize(w, h)` — ресайз до размеров.
- `transform(t)` — трансформация (значение 0..7).
- `opacity(a)` — прозрачность 0..255.
- `remove_alpha` или `noalpha` — убрать альфа-канал.
- `make_alpha(color)` — сделать альфу по цвету (цвет в `#RRGGBB`).
### Цвет и яркость
- `invert(channels="rgb")` — инверсия каналов (`r`, `g`, `b`, `a`).
- `brighten()` — лёгкое осветление.
- `contrast(value, brightness)` — контраст и яркость (-127..127).
- `multiply(color)` — умножение на цвет.
- `screen(color)` — экранный режим.
- `colorize(color, ratio=255)` — тонирование цветом.
### Анимация
`anim` можно использовать в базе (с указанием текстуры) или в пайплайне
над текущим изображением.
База:
```
anim(tex, frame_w, frame_h, frames, fps, smooth, axis)
```
Пайплайн:
```
... |> anim(frame_w, frame_h, frames, fps, smooth, axis)
```
Именованные аргументы:
- `tex` — имя текстуры (только для базового `anim`).
- `frame_w` или `w`
- `frame_h` или `h`
- `frames` или `count`
- `fps`
- `smooth` (0/1)
- `axis` — режим нарезки:
- `g` или пусто: по сетке (слева направо, сверху вниз)
- `x`/`h`: по горизонтали
- `y`/`v`: по вертикали
Если `frames` не задан, количество кадров вычисляется автоматически:
- сетка: `(sheet.W / frame_w) * (sheet.H / frame_h)`
- ось X/Y: `sheet.W / frame_w` или `sheet.H / frame_h`
Примеры:
```
anim("core:sheet", 16, 16, fps=8) # сетка по умолчанию
anim("core:sheet", 16, 16, axis="x") # по горизонтали
stone |> anim(16, 16, fps=10, smooth=1) # анимировать текущую текстуру
```
## Вложенные текстурные выражения
Некоторые операции принимают текстуру в аргументах. Чтобы передать не только имя,
но и полноценную программу, используйте префикс `tex`:
```
overlay( tex "core:stone" |> resize(16,16) |> brighten() )
```

View File

@@ -1,81 +0,0 @@
{
"textures": {
"default": "acacia_planks.png"
},
"cuboids": [
{
"from": [
-0.5,
-0.5,
-0.5
],
"to": [
0.5,
0.5,
0.5
],
"faces": {
"down": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "down"
},
"up": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "up"
},
"north": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "north"
},
"south": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "south"
},
"west": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "west"
},
"east": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "east"
}
}
}
]
}

View File

@@ -1,81 +0,0 @@
{
"textures": {
"default": "frame.png"
},
"cuboids": [
{
"from": [
-0.5,
-0.5,
-0.5
],
"to": [
0.5,
0.5,
0.5
],
"faces": {
"down": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "down"
},
"up": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "up"
},
"north": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "north"
},
"south": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "south"
},
"west": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "west"
},
"east": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "east"
}
}
}
]
}

View File

@@ -1,81 +0,0 @@
{
"textures": {
"default": "grass.png"
},
"cuboids": [
{
"from": [
-0.5,
-0.5,
-0.5
],
"to": [
0.5,
0.5,
0.5
],
"faces": {
"down": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "down"
},
"up": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "up"
},
"north": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "north"
},
"south": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "south"
},
"west": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "west"
},
"east": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "east"
}
}
}
]
}

View File

@@ -1,81 +0,0 @@
{
"textures": {
"default": "jungle_planks.png"
},
"cuboids": [
{
"from": [
-0.5,
-0.5,
-0.5
],
"to": [
0.5,
0.5,
0.5
],
"faces": {
"down": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "down"
},
"up": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "up"
},
"north": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "north"
},
"south": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "south"
},
"west": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "west"
},
"east": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "east"
}
}
}
]
}

View File

@@ -1,81 +0,0 @@
{
"textures": {
"default": "oak_planks.png"
},
"cuboids": [
{
"from": [
-0.5,
-0.5,
-0.5
],
"to": [
0.5,
0.5,
0.5
],
"faces": {
"down": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "down"
},
"up": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "up"
},
"north": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "north"
},
"south": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "south"
},
"west": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "west"
},
"east": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "east"
}
}
}
]
}

View File

@@ -1,81 +0,0 @@
{
"textures": {
"default": "tropical_rainforest_wood.png"
},
"cuboids": [
{
"from": [
-0.5,
-0.5,
-0.5
],
"to": [
0.5,
0.5,
0.5
],
"faces": {
"down": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "down"
},
"up": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "up"
},
"north": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "north"
},
"south": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "south"
},
"west": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "west"
},
"east": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "east"
}
}
}
]
}

View File

@@ -1,81 +0,0 @@
{
"textures": {
"default": "willow_wood.png"
},
"cuboids": [
{
"from": [
-0.5,
-0.5,
-0.5
],
"to": [
0.5,
0.5,
0.5
],
"faces": {
"down": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "down"
},
"up": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "up"
},
"north": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "north"
},
"south": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "south"
},
"west": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "west"
},
"east": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "east"
}
}
}
]
}

View File

@@ -1,81 +0,0 @@
{
"textures": {
"default": "xnether_blue_wood.png"
},
"cuboids": [
{
"from": [
-0.5,
-0.5,
-0.5
],
"to": [
0.5,
0.5,
0.5
],
"faces": {
"down": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "down"
},
"up": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "up"
},
"north": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "north"
},
"south": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "south"
},
"west": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "west"
},
"east": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "east"
}
}
}
]
}

View File

@@ -1,81 +0,0 @@
{
"textures": {
"default": "xnether_purple_wood.png"
},
"cuboids": [
{
"from": [
-0.5,
-0.5,
-0.5
],
"to": [
0.5,
0.5,
0.5
],
"faces": {
"down": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "down"
},
"up": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "up"
},
"north": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "north"
},
"south": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "south"
},
"west": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "west"
},
"east": {
"uv": [
0,
0,
1,
1
],
"texture": "default",
"cullface": "east"
}
}
}
]
}

View File

@@ -1,14 +0,0 @@
{
"meta==0": {
"model": "node/grass.json"
},
"meta==1": {
"model": "node/oak_planks.json"
},
"meta==2": {
"model": "node/jungle_planks.json"
},
"meta==3": {
"model": "node/acacia_planks.json"
}
}

View File

@@ -1,14 +0,0 @@
{
"meta==0": {
"model": "node/tropical_rainforest_wood.json"
},
"meta==1": {
"model": "node/willow_wood.json"
},
"meta==2": {
"model": "node/xnether_blue_wood.json"
},
"meta==3": {
"model": "node/xnether_purple_wood.json"
}
}

View File

@@ -1,14 +0,0 @@
{
"meta==0": {
"model": "node/frame.json"
},
"meta==1": {
"model": "node/grass.json"
},
"meta==2": {
"model": "node/oak_planks.json"
},
"meta==3": {
"model": "node/acacia_planks.json"
}
}

View File

@@ -1,14 +0,0 @@
{
"meta==0": {
"model": "node/jungle_planks.json"
},
"meta==1": {
"model": "node/tropical_rainforest_wood.json"
},
"meta==2": {
"model": "node/willow_wood.json"
},
"meta==3": {
"model": "node/xnether_blue_wood.json"
}
}

View File

@@ -1,14 +0,0 @@
{
"meta==0": {
"model": "node/oak_planks.json"
},
"meta==1": {
"model": "node/jungle_planks.json"
},
"meta==2": {
"model": "node/acacia_planks.json"
},
"meta==3": {
"model": "node/willow_wood.json"
}
}

View File

@@ -1,14 +0,0 @@
{
"meta==0": {
"model": "node/grass.json"
},
"meta==1": {
"model": "node/frame.json"
},
"meta==2": {
"model": "node/xnether_purple_wood.json"
},
"meta==3": {
"model": "node/tropical_rainforest_wood.json"
}
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 138 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.8 KiB

View File

@@ -1,98 +0,0 @@
-- parent = default:air
--
-- hasHalfTransparency
-- collideBox = {}
-- plantLike = {}
-- nodebox = {}
local node_template = {
parent = "default:normal" or node_template,
render = {
has_half_transparency = false
},
collision = {
},
events = {
},
node_advancement_factory = function(world_id, node_pos)
local node_advancement = {
onLoad = function(data)
end,
onSave = function()
return {}
end
}
return node_advancement
end
}
local instance = {}
--[[
Движок автоматически подгружает ассеты из папки assets
В этом методе можно зарегистрировать ассеты из иных источников
Состояния нод, частицы, анимации, модели, текстуры, звуки, шрифты
]]--
function instance.assetsInit()
end
--[[
*preInit. События для регистрации определений игрового контента
Ноды, воксели, миры, порталы, сущности, предметы
]]--
function instance.lowPreInit()
end
--[[
До вызова preInit будет выполнена регистрация
контента из файлов в папке content
]]--
function instance.preInit()
local node_air = {}
node_air.hasHalfTransparency = false
node_air.collideBox = nil
node_air.render = nil
core.register_node('test0', {})
core.register_node('test1', {})
core.register_node('test2', {})
core.register_node('test3', {})
core.register_node('test4', {})
core.register_node('test5', {})
end
function instance.highPreInit()
end
--[[
На этом этапе можно наложить изменения
на зарегистрированный другими модами контент
]]--
function instance.init()
end
function instance.postInit()
end
function instance.preDeInit()
end
function instance.deInit()
end
function instance.postDeInit()
core.unregister_node('test0')
core.unregister_node('test1')
core.unregister_node('test2')
core.unregister_node('test3')
core.unregister_node('test4')
core.unregister_node('test5')
end
return instance

View File

@@ -1,9 +0,0 @@
{
"id": "test",
"name": "Test Mod",
"description": "Это тестовый мод",
"depends": [],
"optional_depends": [],
"author": "DrSocalkwe3n",
"version": [0, 0, 0, 1]
}