From 4ab8dd170fc9efa382942e2db314666e9a6d50c5 Mon Sep 17 00:00:00 2001 From: DronCode Date: Fri, 29 Jan 2021 16:43:03 +0300 Subject: [PATCH 1/7] GMSTool: * Added new component PRPVFSM - PRP Visitor Finite State Machine. It's a top controller of PRP interpreter core. Now this component reads all ZDefines from PRP and storing it into the map between name and value. --- Tools/GMSInfo/include/BinaryWalker.h | 9 +- Tools/GMSInfo/include/PRP/PRPTypes.h | 14 ++ Tools/GMSInfo/include/PRP/PRPVFSM.h | 121 ++++++++++ Tools/GMSInfo/source/PRP/PRP.cpp | 271 ++++++++++----------- Tools/GMSInfo/source/PRP/PRPVFSM.cpp | 337 +++++++++++++++++++++++++++ 5 files changed, 617 insertions(+), 135 deletions(-) create mode 100644 Tools/GMSInfo/include/PRP/PRPVFSM.h create mode 100644 Tools/GMSInfo/source/PRP/PRPVFSM.cpp diff --git a/Tools/GMSInfo/include/BinaryWalker.h b/Tools/GMSInfo/include/BinaryWalker.h index f6170e6..05de76c 100644 --- a/Tools/GMSInfo/include/BinaryWalker.h +++ b/Tools/GMSInfo/include/BinaryWalker.h @@ -29,6 +29,7 @@ namespace ReGlacier public: virtual ~IBaseStreamWalker() noexcept = default; + IBaseStreamWalker() = default; IBaseStreamWalker(size_t size, size_t offset); IBaseStreamWalker(const IBaseStreamWalker& copy); @@ -104,9 +105,10 @@ namespace ReGlacier class BinaryWalker final : public IBaseStreamWalker { private: - uint8_t* m_buffer; + uint8_t* m_buffer { nullptr }; public: + BinaryWalker() = default; BinaryWalker(uint8_t* buffer, size_t size); BinaryWalker(const BinaryWalker& copy) noexcept; @@ -184,9 +186,10 @@ namespace ReGlacier if (!m_buffer || m_offset + sizeof(T) * size >= m_size) throw std::out_of_range { "Unable to read buffer. Not enough bytes" }; - std::memcpy(buffer, m_buffer + (sizeof(T) * size), size); + const size_t offset = sizeof(T) * size; + std::memcpy(buffer, m_buffer + m_offset, offset); - m_offset += (sizeof(T) * size); + m_offset += offset; } /** diff --git a/Tools/GMSInfo/include/PRP/PRPTypes.h b/Tools/GMSInfo/include/PRP/PRPTypes.h index 01441df..7370489 100644 --- a/Tools/GMSInfo/include/PRP/PRPTypes.h +++ b/Tools/GMSInfo/include/PRP/PRPTypes.h @@ -5,6 +5,20 @@ namespace ReGlacier { +#pragma pack(push, 1) + struct SPRP + { + static constexpr size_t kMagicSize = 0xE; + + char magic[kMagicSize]; //0x0 + bool bIsRaw; //0xE + uint32_t bFlags; //0xF + uint32_t __Unknown__; //0x13; + uint32_t totalKeysCount; //0x17 + uint32_t dataOffset; //0x1B + }; +#pragma pack(pop) + enum PRP_ETag : uint8_t { TAG_Array = 0x1, diff --git a/Tools/GMSInfo/include/PRP/PRPVFSM.h b/Tools/GMSInfo/include/PRP/PRPVFSM.h new file mode 100644 index 0000000..12b9e3a --- /dev/null +++ b/Tools/GMSInfo/include/PRP/PRPVFSM.h @@ -0,0 +1,121 @@ +/** + * @file PRPVFSM + * + * PRP File format Visitor FSM + */ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace ReGlacier +{ + class IPRPVisitor; + using PRPToken = uint32_t; + + struct ZToken + { + static constexpr PRPToken Void = 0x0FFFFFFFF; + static constexpr PRPToken Unknown = 0x0FFFFFFFE; + static constexpr PRPToken Joker = 0x0FFFFFFFD; + }; + + enum EPropertyType : unsigned int + { + Type_14 = 0xE, + Type_10 = 0xA, + Type_9 = 0x9, + Type_8 = 0x8, + Type_7 = 0x7, + Type_2 = 0x2, + Type_1 = 0x1 + }; + + class PRPWalker + { + friend class IPRPVisitor; + public: + enum Flags : unsigned int { + NO_FLAG = 0x0, + END_OF_STREAM = 1 << 1, + ERROR_IN_STREAM = 1 << 2 + }; + + using ZString = std::string; + using ZRefTab = std::vector; + using ZI32Vec = std::vector; + using ZF32Vec = std::vector; + + using ZDefine_t = std::variant< + bool, + uint8_t, + uint16_t, + uint32_t, + float, + double, + ZString, + ZI32Vec, + ZF32Vec, + ZRefTab>; + + private: + std::unique_ptr m_buffer; + unsigned int m_flags { Flags::NO_FLAG }; + size_t m_bufferSize { 0 }; + size_t m_keysCount { 0 }; + size_t m_valuesOffset { 0 }; + std::vector m_tokenTable {}; + std::unordered_map m_zDefines {}; + SPRP m_header { 0 }; + BinaryWalker m_walker {}; + + public: + PRPWalker(std::unique_ptr&& buffer, size_t bufferSize); + + [[nodiscard]] bool IsEOS() const { return m_flags & Flags::END_OF_STREAM; }; + [[nodiscard]] bool HasErrors() const { return m_flags & Flags::ERROR_IN_STREAM; } + + void Prepare(IPRPVisitor* visitor); + + private: + bool LoadZDefines(IPRPVisitor* visitor); + + private: + uint8_t GetCurrentByte() const; + size_t GetCurrentOffset() const; + }; + + class IPRPVisitor + { + public: + virtual ~IPRPVisitor() = default; + + virtual void ExchangeHeader(PRPWalker* walker, unsigned int type); + virtual void ExchangeFooter(PRPWalker* walker, unsigned int type); + virtual void ExchangeContainer(PRPWalker* walker, unsigned int* capacity); + + virtual void BeginArray(PRPWalker* walker, PRPToken token, unsigned int size); + virtual void EndArray(PRPWalker* walker); + + virtual void Visit_I8(PRPWalker* walker, PRPToken token, uint8_t& result); + virtual void Visit_I16(PRPWalker* walker, PRPToken token, uint16_t& result); + virtual void Visit_I32(PRPWalker* walker, PRPToken token, uint32_t& result); + virtual void Visit_F32(PRPWalker* walker, PRPToken token, float& result); + virtual void Visit_F64(PRPWalker* walker, PRPToken token, double& result); + virtual void Visit_String(PRPWalker* walker, PRPToken token, std::string& result); + + virtual void Visit_EndOfStream(PRPWalker* walker); + + virtual void Visit_Unrecognized(PRPWalker* walker, size_t offset, uint8_t byte); + + template void ExchangeArray(PRPWalker* walker, PRPToken token, T* data, unsigned int size); + template <> void ExchangeArray(PRPWalker* walker, PRPToken token, uint32_t* data, unsigned int size); + template <> void ExchangeArray(PRPWalker* walker, PRPToken token, float* data, unsigned int size); + }; +} \ No newline at end of file diff --git a/Tools/GMSInfo/source/PRP/PRP.cpp b/Tools/GMSInfo/source/PRP/PRP.cpp index 95ab14b..241e857 100644 --- a/Tools/GMSInfo/source/PRP/PRP.cpp +++ b/Tools/GMSInfo/source/PRP/PRP.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -112,147 +113,153 @@ namespace ReGlacier return false; } - BinaryWalker binaryWalker { prpBuffer.get(), prpBufferSize }; - - { - std::string headerStr = binaryWalker.ReadZString(strlen(kExpectedIdentifier)); - - if (std::string_view(headerStr) != std::string_view(kExpectedIdentifier)) - { - spdlog::error("PRP::Load| Failed to load PRP {}. Reason: bad header!", m_name); - return false; - } - - spdlog::info("PRP::Load| Header is OK"); - binaryWalker.Seek(kKeysOffset, IBaseStreamWalker::BEGIN); - } - - { - m_keysCount = binaryWalker.ReadUInt32(); - m_valuesOffset = binaryWalker.ReadUInt32(); - - if (m_valuesOffset >= prpBufferSize) - { - spdlog::error("PRP::Load| Failed to load PRP {}. Reason: Bad 'values' offset!", m_name); - m_keysCount = 0; - m_valuesOffset = 0; - return false; - } - } - - spdlog::info("PRP::Load| Header looks OK."); - spdlog::info("PRP::Load| Total keys: {}", m_keysCount); - - m_keys.clear(); - m_keys.reserve(m_keysCount); - - while (m_keys.size() != m_keysCount) - { - std::string val = binaryWalker.ReadZString(); - if (!val.empty()) - { - m_keys.push_back(val); - } - } - - // Decompile it - TryToDecompileEntities(std::move(prpBuffer), prpBufferSize); + IPRPVisitor* visitor = new IPRPVisitor(); + PRPWalker walker { std::move(prpBuffer), prpBufferSize }; + walker.Prepare(visitor); + + delete visitor; + +// BinaryWalker binaryWalker { prpBuffer.get(), prpBufferSize }; +// +// { +// std::string headerStr = binaryWalker.ReadZString(strlen(kExpectedIdentifier)); +// +// if (std::string_view(headerStr) != std::string_view(kExpectedIdentifier)) +// { +// spdlog::error("PRP::Load| Failed to load PRP {}. Reason: bad header!", m_name); +// return false; +// } +// +// spdlog::info("PRP::Load| Header is OK"); +// binaryWalker.Seek(kKeysOffset, IBaseStreamWalker::BEGIN); +// } +// +// { +// m_keysCount = binaryWalker.ReadUInt32(); +// m_valuesOffset = binaryWalker.ReadUInt32(); +// +// if (m_valuesOffset >= prpBufferSize) +// { +// spdlog::error("PRP::Load| Failed to load PRP {}. Reason: Bad 'values' offset!", m_name); +// m_keysCount = 0; +// m_valuesOffset = 0; +// return false; +// } +// } +// +// spdlog::info("PRP::Load| Header looks OK."); +// spdlog::info("PRP::Load| Total keys: {}", m_keysCount); +// +// m_keys.clear(); +// m_keys.reserve(m_keysCount); +// +// while (m_keys.size() != m_keysCount) +// { +// std::string val = binaryWalker.ReadZString(); +// if (!val.empty()) +// { +// m_keys.push_back(val); +// } +// } +// +// // Decompile it +// TryToDecompileEntities(std::move(prpBuffer), prpBufferSize); return true; } void PRP::PrintInfo() { - spdlog::info("PRP Info: "); - spdlog::info("Keys: {}", m_keysCount); - int index = 0; - for (index = 0; index < m_keysCount; index++) - { - spdlog::info(" [{}] = {}", index, m_keys[index]); - } - spdlog::info(" --- END OF PRP --- "); +// spdlog::info("PRP Info: "); +// spdlog::info("Keys: {}", m_keysCount); +// int index = 0; +// for (index = 0; index < m_keysCount; index++) +// { +// spdlog::info(" [{}] = {}", index, m_keys[index]); +// } +// spdlog::info(" --- END OF PRP --- "); } void PRP::TryToDecompileEntities(std::unique_ptr&& buffer, size_t bufferSize) { - BinaryWalker binaryWalker { buffer.get(), bufferSize }; - binaryWalker.Seek(kKeysListOffset, IBaseStreamWalker::BEGIN); - binaryWalker.Seek(m_valuesOffset, IBaseStreamWalker::CURR); - spdlog::info("Current pos: {}", binaryWalker.GetPosition()); - - // Iterate over bytes, try to find TAG, if tag occured - decompress it's data if we can - PRP_ETag tag = PRP_ETag::NO_TAG; - int tagId = 0; - - if (m_tree) - { - delete m_tree; - m_tree = nullptr; - } - - m_tree = new PRPTreeNode(nullptr, PRP_ETag::NO_TAG); // No tag at root - auto currentNode = m_tree; - - do - { - auto pos = binaryWalker.GetPosition(); - uint8_t byte = binaryWalker.ReadUInt8(); - tag = PRP_ETag_Helpers::FromByte(byte); - - if (tag == PRP_ETag::NO_TAG) continue; - const auto tagName = PRP_ETag_Helpers::ToString(tag); - - // Recognize tag flags - if (PRP_ETag_Helpers::IsTagIncreaseDepthLevel(tag)) - { - // Allocate new depth level, insert as child, set current not to new depth controller - auto newChild = new PRPTreeNode(currentNode, tag); - currentNode->AddChild(newChild); - newChild->SetRefToBuffer(pos); - currentNode = newChild; - } - else if (PRP_ETag_Helpers::IsTagDecreateDepthLevel(tag)) - { - // Change depth level of tree - jump to parent node - if (!currentNode->IsRoot()) - { - auto lastChild = new PRPTreeNode(currentNode, tag); - lastChild->SetRefToBuffer(pos); - currentNode->AddChild(lastChild); - currentNode = currentNode->GetParent(); - } - else - { - spdlog::warn(" *** TREE TRAVELING WARNING: TAG {} AT +{:X} TOLD US THAT WE NEED TO JUMP UP, BUT NO WAY TO DO THIS! *** ", PRP_ETag_Helpers::ToString(tag), pos); - } - } - else - { - // Normal tag, insert as child - auto child = new PRPTreeNode(currentNode, tag); - child->SetRefToBuffer(pos); - currentNode->AddChild(child); - } - - if (tag == PRP_ETag::TAG_Float32) - { - auto fvl = binaryWalker.Read(); - spdlog::info("[{}] +{:X} {} = {}", tagId, pos, tagName, fvl); - } - else if (tag == PRP_ETag::TAG_Int32) - { - auto i32 = binaryWalker.Read(); - spdlog::info("[{}] +{:X} {} = {} (hex 0x{:08X})", tagId, pos, tagName, i32, i32); - } - else - { - spdlog::info("[{}] +{:X} {} = ?", tagId, pos, tagName); - } - - ++tagId; - } - while (tag != PRP_ETag::TAG_EndOfStream); - - spdlog::info("Total ETags: {}, Total Keys: {}", tagId + 1, m_keysCount); +// BinaryWalker binaryWalker { buffer.get(), bufferSize }; +// binaryWalker.Seek(kKeysListOffset, IBaseStreamWalker::BEGIN); +// binaryWalker.Seek(m_valuesOffset, IBaseStreamWalker::CURR); +// spdlog::info("Current pos: {}", binaryWalker.GetPosition()); + +// // Iterate over bytes, try to find TAG, if tag occured - decompress it's data if we can +// PRP_ETag tag = PRP_ETag::NO_TAG; +// int tagId = 0; +// +// if (m_tree) +// { +// delete m_tree; +// m_tree = nullptr; +// } +// +// m_tree = new PRPTreeNode(nullptr, PRP_ETag::NO_TAG); // No tag at root +// auto currentNode = m_tree; +// +// do +// { +// auto pos = binaryWalker.GetPosition(); +// uint8_t byte = binaryWalker.ReadUInt8(); +// tag = PRP_ETag_Helpers::FromByte(byte); +// +// if (tag == PRP_ETag::NO_TAG) continue; +// const auto tagName = PRP_ETag_Helpers::ToString(tag); +// +// // Recognize tag flags +// if (PRP_ETag_Helpers::IsTagIncreaseDepthLevel(tag)) +// { +// // Allocate new depth level, insert as child, set current not to new depth controller +// auto newChild = new PRPTreeNode(currentNode, tag); +// currentNode->AddChild(newChild); +// newChild->SetRefToBuffer(pos); +// currentNode = newChild; +// } +// else if (PRP_ETag_Helpers::IsTagDecreateDepthLevel(tag)) +// { +// // Change depth level of tree - jump to parent node +// if (!currentNode->IsRoot()) +// { +// auto lastChild = new PRPTreeNode(currentNode, tag); +// lastChild->SetRefToBuffer(pos); +// currentNode->AddChild(lastChild); +// currentNode = currentNode->GetParent(); +// } +// else +// { +// spdlog::warn(" *** TREE TRAVELING WARNING: TAG {} AT +{:X} TOLD US THAT WE NEED TO JUMP UP, BUT NO WAY TO DO THIS! *** ", PRP_ETag_Helpers::ToString(tag), pos); +// } +// } +// else +// { +// // Normal tag, insert as child +// auto child = new PRPTreeNode(currentNode, tag); +// child->SetRefToBuffer(pos); +// currentNode->AddChild(child); +// } +// +// if (tag == PRP_ETag::TAG_Float32) +// { +// auto fvl = binaryWalker.Read(); +// spdlog::info("[{}] +{:X} {} = {}", tagId, pos, tagName, fvl); +// } +// else if (tag == PRP_ETag::TAG_Int32) +// { +// auto i32 = binaryWalker.Read(); +// spdlog::info("[{}] +{:X} {} = {} (hex 0x{:08X})", tagId, pos, tagName, i32, i32); +// } +// else +// { +// spdlog::info("[{}] +{:X} {} = ?", tagId, pos, tagName); +// } +// +// ++tagId; +// } +// while (tag != PRP_ETag::TAG_EndOfStream); +// +// spdlog::info("Total ETags: {}, Total Keys: {}", tagId + 1, m_keysCount); } } diff --git a/Tools/GMSInfo/source/PRP/PRPVFSM.cpp b/Tools/GMSInfo/source/PRP/PRPVFSM.cpp new file mode 100644 index 0000000..5da48d6 --- /dev/null +++ b/Tools/GMSInfo/source/PRP/PRPVFSM.cpp @@ -0,0 +1,337 @@ +#include +#include +#include +#include +#include + +namespace ReGlacier +{ + template <> + struct BinaryWalkerADL + { + static void Read(const BinaryWalker& binaryWalker, SPRP& value) + { + binaryWalker.ReadArray(&value.magic[0], SPRP::kMagicSize); + value.bIsRaw = static_cast(binaryWalker.ReadUInt8()); + value.bFlags = binaryWalker.ReadUInt32(); + value.__Unknown__ = binaryWalker.ReadUInt32(); + value.totalKeysCount = binaryWalker.ReadUInt32(); + value.dataOffset = binaryWalker.ReadUInt32(); + } + + static void Write(BinaryWalker& binaryWalker, const SPRP& value) + { + spdlog::error("NOT IMPLEMENTED"); + assert(false); + } + }; + + static constexpr const char* kExpectedIdentifier = "IOPacked v0.1"; + static constexpr int kKeysOffset = 0x17; + + PRPWalker::PRPWalker(std::unique_ptr&& buffer, size_t bufferSize) + : m_buffer(std::move(buffer)) + , m_bufferSize(bufferSize) + , m_flags(Flags::NO_FLAG) + { + } + + void PRPWalker::Prepare(IPRPVisitor* visitor) + { + if (!visitor) { + assert(visitor != nullptr); + return; + } + + m_walker = BinaryWalker { m_buffer.get(), m_bufferSize }; + BinaryWalkerADL::Read(m_walker, m_header); + + if (std::string_view { m_header.magic } != std::string_view { kExpectedIdentifier }) { + m_flags |= Flags::ERROR_IN_STREAM; + spdlog::error("PRPWalker| Bad PRP file (magic mismatch!)"); + return; + } + + if (m_header.totalKeysCount >= 0xDEADBEEF) { + m_flags |= Flags::ERROR_IN_STREAM; + spdlog::error("PRPWalker| Bad PRP file (too much tokens in table"); + return; + } + + if (m_header.dataOffset >= m_bufferSize) { + m_flags |= Flags::ERROR_IN_STREAM; + spdlog::error("PRPWalker| Bad PRP file (bad data offset"); + return; + } + + spdlog::info("PRP looks ok. Reading tokens table"); + m_tokenTable.clear(); + + m_tokenTable.reserve(m_header.totalKeysCount); + + int tokenIndex = 0; + while (tokenIndex < m_header.totalKeysCount) + { + std::string token; + BinaryWalkerADL::Read(m_walker, token); + if (token.empty()) continue; + + m_tokenTable.emplace_back(std::move(token)); + ++tokenIndex; + } + + spdlog::info("Tokens table is ready. Total tokens count is {}", m_header.totalKeysCount); + m_walker.Seek(sizeof(SPRP) + m_header.dataOffset, BinaryWalker::BEGIN); //Reset position + m_walker.Seek(4, BinaryWalker::CURR); //HACK! TODO: Find a way to fix this + + m_zDefines.clear(); + + LoadZDefines(visitor); + spdlog::info("DBG"); + } + + bool PRPWalker::LoadZDefines(IPRPVisitor* visitor) + { + // Load ZDefines + uint32_t capacity { 0 }; + + visitor->ExchangeContainer(this, &capacity); + + for (int i = 0; i < capacity; i++) + { + std::string name; + uint32_t typeId { 0 }; + + /// Get Name + visitor->ExchangeHeader(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + visitor->Visit_String(this, ZToken::Void, name); + visitor->ExchangeFooter(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + + /// Get Type ID + visitor->ExchangeHeader(this, EPropertyType::Type_8); + visitor->Visit_I32(this, ZToken::Void, typeId); + visitor->ExchangeFooter(this, EPropertyType::Type_8); + + ZDefine_t value; + + /// Read internal data + switch (typeId) + { + case 2: + { + uint32_t size { 0 }; + + // Read Size + visitor->ExchangeHeader(this, EPropertyType::Type_8); + visitor->Visit_I32(this, ZToken::Void, size); + visitor->ExchangeFooter(this, EPropertyType::Type_8); + + // Read Data + ZI32Vec vec; + vec.resize(size); + visitor->ExchangeArray(this, ZToken::Void, vec.data(), size); + + value = vec; + } + break; + case 3: + { + uint32_t size { 0 }; + + // Read Size + visitor->ExchangeHeader(this, EPropertyType::Type_8); + visitor->Visit_I32(this, ZToken::Void, size); + visitor->ExchangeFooter(this, EPropertyType::Type_8); + + // Read Data + ZF32Vec vec; + vec.resize(size); + visitor->ExchangeArray(this, ZToken::Void, vec.data(), size); + + value = vec; + } + break; + case 0xC: + case 0xE: + case 0x10: + { + std::string geomRefStr; + + visitor->ExchangeHeader(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + visitor->Visit_String(this, ZToken::Void, geomRefStr); + visitor->ExchangeFooter(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + + value = geomRefStr; + } + break; + case 0x11: + { + unsigned int elementsNr { 0 }; + + visitor->ExchangeContainer(this, &elementsNr); + std::vector reftab; + reftab.reserve(elementsNr); + + for (int j = 0; j < elementsNr; j++) + { + std::string geomName; + visitor->ExchangeHeader(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + visitor->Visit_String(this, ZToken::Void, geomName); + visitor->ExchangeFooter(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + + reftab.push_back(geomName); + } + + value = reftab; + } + break; + default: + spdlog::warn("Unknown type id {}", typeId); + assert(false); + break; + } + + m_zDefines[name] = value; + } + + return true; + } + + uint8_t PRPWalker::GetCurrentByte() const { + return m_walker.ReadInt8(); + } + + size_t PRPWalker::GetCurrentOffset() const { + return m_walker.GetPosition(); + } + + /// --------------------------------------------------------------------------------------------------------- + + void IPRPVisitor::ExchangeHeader(PRPWalker* walker, unsigned int /*type*/) + { + walker->m_walker.ReadUInt8(); + } + + void IPRPVisitor::ExchangeFooter(PRPWalker* walker, unsigned int type) + { + if (type == EPropertyType::Type_14) { + if (walker->m_header.bFlags & 1) { + walker->m_walker.ReadUInt8(); + } + } + } + + void IPRPVisitor::ExchangeContainer(PRPWalker* walker, unsigned int* capacity) + { + ExchangeHeader(walker, EPropertyType::Type_10); + + const uint32_t res = walker->m_walker.ReadUInt32(); + if (capacity) { + *capacity = res; + } + } + + void IPRPVisitor::BeginArray(PRPWalker* walker, PRPToken token, unsigned int size) + { + if (!(walker->m_header.bFlags & 1)) + return; + + walker->m_walker.ReadUInt8(); + + if (walker->m_header.bFlags & 1) + { + walker->m_walker.ReadUInt32(); + //TODO: Maybe we should check size constrains here? (walker->m_walker.ReadUInt32() == size) + } + } + + void IPRPVisitor::EndArray(PRPWalker* walker) + { + walker->m_walker.ReadUInt8(); + } + + void IPRPVisitor::Visit_I8(PRPWalker* walker, PRPToken token, uint8_t& result) + { + result = walker->GetCurrentByte(); + } + + void IPRPVisitor::Visit_I16(PRPWalker* walker, PRPToken token, uint16_t& result) + { + result = walker->m_walker.ReadUInt16(); + } + + void IPRPVisitor::Visit_I32(PRPWalker* walker, PRPToken token, uint32_t& result) + { + result = walker->m_walker.ReadUInt32(); + } + + void IPRPVisitor::Visit_F32(PRPWalker* walker, PRPToken token, float& result) + { + result = walker->m_walker.Read(); + } + + void IPRPVisitor::Visit_F64(PRPWalker* walker, PRPToken token, double& result) + { + result = walker->m_walker.Read(); + } + + void IPRPVisitor::Visit_String(PRPWalker* walker, PRPToken token, std::string& result) + { + const uint32_t index = walker->m_walker.ReadUInt32(); + if (index > walker->m_tokenTable.size()) { + spdlog::error("IPRPVisitor::Visit_String| Index of string is out of bounds! Index is {:8X}", index); + return; + } + + result = walker->m_tokenTable[index]; + } + + void IPRPVisitor::Visit_EndOfStream(PRPWalker* walker) + { + walker->m_flags |= PRPWalker::Flags::END_OF_STREAM; + spdlog::info("IPRPVisitor| End of stream"); + } + + void IPRPVisitor::Visit_Unrecognized(PRPWalker* walker, size_t offset, uint8_t byte) + { + spdlog::warn("IPRPVisitor| Unrecognized byte of context. Byte {:2X} at +0x{:8X}", byte, offset); + } + + template<> + void IPRPVisitor::ExchangeArray(PRPWalker* walker, PRPToken token, uint32_t* data, unsigned int size) + { + BeginArray(walker, token, size); + + auto sz = size; + uint32_t* pCurrent = data; + do { + ExchangeHeader(walker, EPropertyType::Type_7); + Visit_I32(walker, token, reinterpret_cast(*pCurrent)); + ExchangeFooter(walker, EPropertyType::Type_7); + + ++pCurrent; + --sz; + } while (sz); + + EndArray(walker); + } + + template<> + void IPRPVisitor::ExchangeArray(PRPWalker* walker, PRPToken token, float* data, unsigned int size) + { + BeginArray(walker, token, size); + + auto sz = size; + float* pCurrent = data; + do { + ExchangeHeader(walker, EPropertyType::Type_7); + Visit_F32(walker, token, reinterpret_cast(*pCurrent)); + ExchangeFooter(walker, EPropertyType::Type_7); + + ++pCurrent; + --sz; + } while (sz); + + EndArray(walker); + } +} \ No newline at end of file From 2a66ad268564b4e4371936a7c435a642e9eb4927 Mon Sep 17 00:00:00 2001 From: DronCode Date: Sun, 31 Jan 2021 03:31:01 +0300 Subject: [PATCH 2/7] GMSTool: * First revision of PRP Properties viewer. Now we can iterate over geoms and their properties. --- Tools/GMSInfo/include/GMS/GMS.h | 3 +- Tools/GMSInfo/include/PRP/PRPTypes.h | 8 +- Tools/GMSInfo/include/PRP/PRPVFSM.h | 22 +- Tools/GMSInfo/source/GMS/GMS.cpp | 4 + Tools/GMSInfo/source/PRP/PRP.cpp | 12 +- Tools/GMSInfo/source/PRP/PRPVFSM.cpp | 363 ++++++++++++++++++++++++++- 6 files changed, 392 insertions(+), 20 deletions(-) diff --git a/Tools/GMSInfo/include/GMS/GMS.h b/Tools/GMSInfo/include/GMS/GMS.h index d49c187..cb3e9b1 100644 --- a/Tools/GMSInfo/include/GMS/GMS.h +++ b/Tools/GMSInfo/include/GMS/GMS.h @@ -32,7 +32,8 @@ namespace ReGlacier void PrintInfo() override; [[nodiscard]] const std::vector& GetExcludedAnimations() const; - [[nodiscard]] const std::vector& GetLinkReferences() const; + [[nodiscard,deprecated("Not working, please, use geoms instead of it")]] const std::vector& GetLinkReferences() const; + [[nodiscard]] const std::vector& GetGeoms() const; [[nodiscard]] std::unique_ptr GetUncompressedBuffer(unsigned int& uncompressedSize); private: diff --git a/Tools/GMSInfo/include/PRP/PRPTypes.h b/Tools/GMSInfo/include/PRP/PRPTypes.h index 7370489..51106c8 100644 --- a/Tools/GMSInfo/include/PRP/PRPTypes.h +++ b/Tools/GMSInfo/include/PRP/PRPTypes.h @@ -54,7 +54,13 @@ namespace ReGlacier TAG_NamedRawData = 0x8D, TAG_NameBitfield = 0x8F, TAG_UNKNOWN, //14,16-123,128,142 - NO_TAG + NO_TAG, + + /// My tags + TAG_StringOrArray_E = 0xE, + TAG_StringOrArray_8E = 0x8E, + + TAG_StringArray = 0xF, }; struct PRP_ETag_Helpers diff --git a/Tools/GMSInfo/include/PRP/PRPVFSM.h b/Tools/GMSInfo/include/PRP/PRPVFSM.h index 12b9e3a..a152c68 100644 --- a/Tools/GMSInfo/include/PRP/PRPVFSM.h +++ b/Tools/GMSInfo/include/PRP/PRPVFSM.h @@ -12,10 +12,13 @@ #include #include #include -#include +#include namespace ReGlacier { + class LevelContainer; + class LevelAssets; + class IPRPVisitor; using PRPToken = uint32_t; @@ -71,7 +74,7 @@ namespace ReGlacier size_t m_keysCount { 0 }; size_t m_valuesOffset { 0 }; std::vector m_tokenTable {}; - std::unordered_map m_zDefines {}; + std::map m_zDefines {}; SPRP m_header { 0 }; BinaryWalker m_walker {}; @@ -80,14 +83,18 @@ namespace ReGlacier [[nodiscard]] bool IsEOS() const { return m_flags & Flags::END_OF_STREAM; }; [[nodiscard]] bool HasErrors() const { return m_flags & Flags::ERROR_IN_STREAM; } + [[nodiscard]] uint32_t GetHeaderBFlags() const { return m_header.bFlags; } - void Prepare(IPRPVisitor* visitor); + void Prepare(IPRPVisitor* visitor, LevelContainer* pLevel, LevelAssets* pAssets); private: bool LoadZDefines(IPRPVisitor* visitor); + bool LoadProperties(IPRPVisitor* visitor, LevelContainer* pLevel, LevelAssets* pAssets); private: uint8_t GetCurrentByte() const; + + public: size_t GetCurrentOffset() const; }; @@ -96,6 +103,12 @@ namespace ReGlacier public: virtual ~IPRPVisitor() = default; + /** + * @brief This is debug function + * @note DEBUG ONLY + */ + virtual void OnByte(PRPWalker* walker, uint8_t byte) {} + virtual void ExchangeHeader(PRPWalker* walker, unsigned int type); virtual void ExchangeFooter(PRPWalker* walker, unsigned int type); virtual void ExchangeContainer(PRPWalker* walker, unsigned int* capacity); @@ -109,11 +122,10 @@ namespace ReGlacier virtual void Visit_F32(PRPWalker* walker, PRPToken token, float& result); virtual void Visit_F64(PRPWalker* walker, PRPToken token, double& result); virtual void Visit_String(PRPWalker* walker, PRPToken token, std::string& result); + virtual void Visit_RawBuffer(PRPWalker* walker, PRPToken token, std::shared_ptr& buffer, size_t& bufferSize); virtual void Visit_EndOfStream(PRPWalker* walker); - virtual void Visit_Unrecognized(PRPWalker* walker, size_t offset, uint8_t byte); - template void ExchangeArray(PRPWalker* walker, PRPToken token, T* data, unsigned int size); template <> void ExchangeArray(PRPWalker* walker, PRPToken token, uint32_t* data, unsigned int size); template <> void ExchangeArray(PRPWalker* walker, PRPToken token, float* data, unsigned int size); diff --git a/Tools/GMSInfo/source/GMS/GMS.cpp b/Tools/GMSInfo/source/GMS/GMS.cpp index ba65ae3..bb5f976 100644 --- a/Tools/GMSInfo/source/GMS/GMS.cpp +++ b/Tools/GMSInfo/source/GMS/GMS.cpp @@ -230,6 +230,10 @@ namespace ReGlacier return m_linkRefs; } + const std::vector & GMS::GetGeoms() const { + return m_geoms; + } + std::unique_ptr GMS::GetUncompressedBuffer(unsigned int& uncompressedSize) { return GetRawGMS(uncompressedSize); diff --git a/Tools/GMSInfo/source/PRP/PRP.cpp b/Tools/GMSInfo/source/PRP/PRP.cpp index 241e857..a3ffd1d 100644 --- a/Tools/GMSInfo/source/PRP/PRP.cpp +++ b/Tools/GMSInfo/source/PRP/PRP.cpp @@ -32,7 +32,11 @@ namespace ReGlacier return PRP_ETag::NO_TAG; } - if (byte == 14 || byte == 128 || byte == 142 || (byte >= 16 && byte <= 123)) + if (byte == 0x0E) return PRP_ETag::TAG_StringOrArray_E; + if (byte == 0x8E) return PRP_ETag::TAG_StringOrArray_8E; + if (byte == 0x0F) return PRP_ETag::TAG_StringArray; + + if (byte == 128 || (byte >= 16 && byte <= 123)) { return PRP_ETag::TAG_UNKNOWN; } @@ -113,11 +117,9 @@ namespace ReGlacier return false; } - IPRPVisitor* visitor = new IPRPVisitor(); + IPRPVisitor visitor {}; PRPWalker walker { std::move(prpBuffer), prpBufferSize }; - walker.Prepare(visitor); - - delete visitor; + walker.Prepare(&visitor, m_container, m_assets); // BinaryWalker binaryWalker { prpBuffer.get(), prpBufferSize }; // diff --git a/Tools/GMSInfo/source/PRP/PRPVFSM.cpp b/Tools/GMSInfo/source/PRP/PRPVFSM.cpp index 5da48d6..40996eb 100644 --- a/Tools/GMSInfo/source/PRP/PRPVFSM.cpp +++ b/Tools/GMSInfo/source/PRP/PRPVFSM.cpp @@ -1,8 +1,18 @@ -#include -#include #include +#include +#include #include + +#include + +#include +#include + +#include + #include +#include +#include namespace ReGlacier { @@ -36,7 +46,7 @@ namespace ReGlacier { } - void PRPWalker::Prepare(IPRPVisitor* visitor) + void PRPWalker::Prepare(IPRPVisitor* visitor, LevelContainer* pLevelContainer, LevelAssets* pAssets) { if (!visitor) { assert(visitor != nullptr); @@ -87,6 +97,7 @@ namespace ReGlacier m_zDefines.clear(); LoadZDefines(visitor); + LoadProperties(visitor, pLevelContainer, pAssets); spdlog::info("DBG"); } @@ -197,6 +208,332 @@ namespace ReGlacier return true; } + class PRPWalkController : public IPRPVisitor + { + enum ContextFlag : unsigned int + { + CF_NONE = 0, + CF_EOS = 1 << 0, + CF_ERROR = 1 << 1, + CF_OBJECT = 1 << 2, + CF_ARRAY = 1 << 3, + CF_CONTAINER = 1 << 4 + }; + + unsigned int m_contextFlags { CF_NONE }; + + public: + void OnByte(PRPWalker* walker, uint8_t byte) override + { + if (IsEndOfStream()) return; + + const size_t offset = walker->GetCurrentOffset() - 1; + const PRP_ETag tag = PRP_ETag_Helpers::FromByte(byte); + + + switch (tag) { + case TAG_Array: + case TAG_NamedArray: + { + m_contextFlags |= ContextFlag::CF_ARRAY; + + uint32_t capacity = 0; + + Visit_I32(walker, ZToken::Void, capacity); + assert(capacity > 0); + + spdlog::info("[OP| +{:0X}] Begin array (size {}, opcode {:2X})", offset, capacity, byte); + } + break; + case TAG_BeginObject: + case TAG_BeginNamedObject: + m_contextFlags |= ContextFlag::CF_OBJECT; + spdlog::info("[OP| +{:0X}] Begin object (opcode {:2X})", offset, byte); + break; + case TAG_Container: + case TAG_NamedContainer: + { + m_contextFlags |= ContextFlag::CF_CONTAINER; + uint32_t v = 0; + Visit_I32(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] Begin container <{}> (opcode {:2X})", offset, v, byte); + } + break; + case TAG_EndArray: + m_contextFlags &= ~ContextFlag::CF_ARRAY; + spdlog::info("[OP| +{:0X}] End array", offset); + break; + case TAG_EndObject: + m_contextFlags &= ~(ContextFlag::CF_OBJECT | ContextFlag::CF_CONTAINER); + spdlog::info("[OP| +{:0X}] End object", offset); + break; + case TAG_EndOfStream: + m_contextFlags |= ContextFlag::CF_EOS; + spdlog::info("[OP| +{:0X}] >>> END OF STREAM <<<", offset); + break; + + case TAG_NamedReference: + case TAG_Reference: + spdlog::info("[OP| +{:0X}] Reference (opcode {:2X})", offset, byte); + assert(false); + break; + case TAG_Char: + case TAG_NamedChar: + { + uint8_t v = 0; + Visit_I8(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] Char {:c} (opcode {:2x})", offset, v, byte); + } + break; + case TAG_Bool: + case TAG_NamedBool: + { + uint8_t v = 0; + Visit_I8(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] Bool {} (opcode {:2X})", offset, v, byte); + } + break; + case TAG_Int8: + case TAG_NamedInt8: + { + uint8_t v = 0; + Visit_I8(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] Int8 {} (opcode {:2X})", offset, v, byte); + } + break; + case TAG_Int16: + case TAG_NamedInt16: + { + uint16_t v = 0; + Visit_I16(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] Int16 {} (opcode {:2X})", offset, v, byte); + } + break; + case TAG_Int32: + case TAG_NamedInt32: + { + uint32_t v = 0; + Visit_I32(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] Int32 {} (opcode {:2X})", offset, v, byte); + } + break; + case TAG_Float32: + case TAG_NamedFloat32: + { + float v = .0f; + Visit_F32(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] Float32 {} (opcode {:2X})", offset, v, byte); + } + break; + case TAG_Float64: + case TAG_NamedFloat64: + { + double v = .0f; + Visit_F64(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] Float32 {} (opcode {:2X})", offset, v, byte); + } + break; + case TAG_String: + case TAG_NamedString: + { + std::string v = {}; + Visit_String(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] String: \"{}\" (opcode {:2X})", offset, v, byte); + } + break; + case TAG_RawData: + case TAG_NamedRawData: + { + size_t bufferSize { 0 }; + std::shared_ptr buffer; + + Visit_RawBuffer(walker, ZToken::Void, buffer, bufferSize); + + if (buffer && bufferSize) + { + spdlog::info("[OP| +{:0X}] TAG_RawData (opcode {:2X}). Buffer of size {:08X} bytes", offset, byte, bufferSize); + std::stringstream ss; + + for (size_t i = 0; i < bufferSize; i++) + { + if (i != 0 && i % 8 == 0) { + ss << '\n'; + } + else + { + ss << std::hex << std::setfill('0') << std::setw(2) << static_cast(buffer.get()[i]) << ' '; + } + } + spdlog::info("{}", ss.str()); + } + else + { + spdlog::error("[OP| +{:0X}] TAG_RawData (opcode {:2X}). Bad buffer!", offset, byte); + assert(false); + } + } + break; + case TAG_Bitfield: + case TAG_NameBitfield: + { + uint32_t v { 0 }; + Visit_I32(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] TAG_Bitfield: {} (opcode {:2X})", offset, v, byte); + } + break; + case TAG_SkipMark: + spdlog::info("[OP| +{:0X}] TAG_SkipMark. Just skip current opcode", offset); + break; + /// >>> MY TAGS <<< + case TAG_StringOrArray_E: + case TAG_StringOrArray_8E: + { + if ((walker->GetHeaderBFlags() >> 2) & 1) { + std::string s {}; + Visit_String(walker, ZToken::Void, s); + spdlog::info("[OP| +{:0X}] TAG_StringOrArray. Read as string: {} (opcode {:2X})", offset, s, byte); + } + else + { + uint32_t v { 0 }; + Visit_I32(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] TAG_StringOrArray. Read as array: {} (i32, opcode {:2X})", offset, v, byte); + } + } + break; + case TAG_StringArray: + { + if ((walker->GetHeaderBFlags() >> 2) & 1) + { + if ((walker->GetHeaderBFlags() >> 3) & 1) + { + //Impl #1 + uint32_t capacity { 0 }; + Visit_I32(walker, ZToken::Void, capacity); + + if (capacity == 0) { + spdlog::info("[OP| +{:X}] TAG_StringArray (impl 1, opcode {:2X}). Array of strings: (EMPTY)", offset, byte); + } + else { + spdlog::info("[OP| +{:X}] TAG_StringArray (impl 1, opcode {:2X}). Array of strings ({}): ", offset, byte, capacity); + + for (int i = 0; i < capacity; i++) + { + std::string s; + Visit_String(walker, ZToken::Void, s); + + spdlog::info("\t[{}] = \"{}\"", i, s); + } + } + } + else + { + //Impl #2 (at 0x458CD0) + spdlog::error("[OP| +{:X}] TAG_StringArray (impl 2, opcode {:2X}). Not implemented", offset, byte); + assert(false); +// uint32_t v1 { 0 }; +// Visit_I32(walker, ZToken::Void, v1); ///??????????? +// +// uint32_t capacity { 0 }; +// Visit_I32(walker, ZToken::Void, capacity); +// +// if (capacity > 0) +// { +// uint32_t v2 { 0 }; +// Visit_I32(walker, ZToken::Void, v2); +// +// } + } + } + else + { + uint32_t v { 0 }; + Visit_I32(walker, ZToken::Void, v); + spdlog::info("[OP| +{:0X}] TAG_StringArray (impl 3, opcode {:2X}). Array as I32: {}", offset, byte, v); + } + } + break; + /// ETC + case TAG_UNKNOWN: + { + uint32_t v = 0; + Visit_I32(walker, ZToken::Void, v); + spdlog::warn("[OP| +{:0X}] TAK_UNKNOWN. Read next I32 {} (opcode {:2X})", offset, v, byte); + } + break; + case NO_TAG: + default: + { + m_contextFlags |= ContextFlag::CF_ERROR; + spdlog::error("[OP| +{:X}] Unknown opcode {:2X}", offset, byte); + assert(false); + } + break; + } + } + + void Reset() + { + m_contextFlags = ContextFlag::CF_NONE; + } + + [[nodiscard]] bool IsInObject() const { + return m_contextFlags & ContextFlag::CF_OBJECT; + } + + [[nodiscard]] bool IsInArray() const { + return m_contextFlags & ContextFlag::CF_ARRAY; + } + + [[nodiscard]] bool IsInContainer() const { + return m_contextFlags & ContextFlag::CF_CONTAINER; + } + + [[nodiscard]] bool IsEndOfStream() const { + return m_contextFlags & ContextFlag::CF_EOS; + } + + [[nodiscard]] bool IsError() const { + return m_contextFlags & ContextFlag::CF_ERROR; + } + }; + + bool PRPWalker::LoadProperties(IPRPVisitor* visitor, LevelContainer* pLevel, LevelAssets* pAssets) + { + /** + * @note this is test code, please refactor this + * @todo REFACTOR THIS LATER + */ + auto gms = std::make_unique(pAssets->GMS, pLevel, pAssets); + if (!gms->Load()) + { + spdlog::error("PRPWalker| Failed to load properties (unable to load GMS)"); + return false; + } + + auto geoms = gms->GetGeoms(); + PRPWalkController controller {}; + + for (const auto& currentGeom : geoms) + { + spdlog::info("Processing properties for geom {} ({:0X})", currentGeom.groupName, currentGeom.id); + do + { + controller.OnByte(this, m_walker.ReadUInt8()); + } + while ((controller.IsInContainer() || controller.IsInObject() || controller.IsInArray()) && !controller.IsEndOfStream() && !controller.IsError()); + + assert(!controller.IsError()); + + if (controller.IsEndOfStream()) + break; + + controller.Reset(); + } + + return true; + } + uint8_t PRPWalker::GetCurrentByte() const { return m_walker.ReadInt8(); } @@ -286,17 +623,27 @@ namespace ReGlacier result = walker->m_tokenTable[index]; } + void IPRPVisitor::Visit_RawBuffer(PRPWalker* walker, + PRPToken token, + std::shared_ptr& buffer, + size_t& bufferSize) { + uint32_t size { 0 }; + + Visit_I32(walker, token, size); + if (size) { + bufferSize = size; + + buffer = std::make_shared(size); + walker->m_walker.ReadArray(buffer.get(), bufferSize); + } + } + void IPRPVisitor::Visit_EndOfStream(PRPWalker* walker) { walker->m_flags |= PRPWalker::Flags::END_OF_STREAM; spdlog::info("IPRPVisitor| End of stream"); } - void IPRPVisitor::Visit_Unrecognized(PRPWalker* walker, size_t offset, uint8_t byte) - { - spdlog::warn("IPRPVisitor| Unrecognized byte of context. Byte {:2X} at +0x{:8X}", byte, offset); - } - template<> void IPRPVisitor::ExchangeArray(PRPWalker* walker, PRPToken token, uint32_t* data, unsigned int size) { From c5d69fd6746cd332f54920c00e117e04e93cb4f2 Mon Sep 17 00:00:00 2001 From: DronCode Date: Sun, 31 Jan 2021 15:06:11 +0300 Subject: [PATCH 3/7] GMSTool: * Fixed bug with token table loader (empty strings are allowed) * Removed assert from TAG_Reference processing routine * Fixed iteration loop (iterate over bytes, not over geoms, TODO: Remove geoms loader code) * Fixed Visit_String implementation * Added Visit_RawContents method --- Tools/GMSInfo/include/PRP/PRPVFSM.h | 1 + Tools/GMSInfo/source/PRP/PRPVFSM.cpp | 58 +++++++++++++++++----------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/Tools/GMSInfo/include/PRP/PRPVFSM.h b/Tools/GMSInfo/include/PRP/PRPVFSM.h index a152c68..4133dac 100644 --- a/Tools/GMSInfo/include/PRP/PRPVFSM.h +++ b/Tools/GMSInfo/include/PRP/PRPVFSM.h @@ -123,6 +123,7 @@ namespace ReGlacier virtual void Visit_F64(PRPWalker* walker, PRPToken token, double& result); virtual void Visit_String(PRPWalker* walker, PRPToken token, std::string& result); virtual void Visit_RawBuffer(PRPWalker* walker, PRPToken token, std::shared_ptr& buffer, size_t& bufferSize); + virtual void Visit_RawContents(PRPWalker* walker, PRPToken token, char* buffer, size_t bufferSize); virtual void Visit_EndOfStream(PRPWalker* walker); diff --git a/Tools/GMSInfo/source/PRP/PRPVFSM.cpp b/Tools/GMSInfo/source/PRP/PRPVFSM.cpp index 40996eb..0bfcdb9 100644 --- a/Tools/GMSInfo/source/PRP/PRPVFSM.cpp +++ b/Tools/GMSInfo/source/PRP/PRPVFSM.cpp @@ -80,11 +80,15 @@ namespace ReGlacier m_tokenTable.reserve(m_header.totalKeysCount); int tokenIndex = 0; - while (tokenIndex < m_header.totalKeysCount) + while (tokenIndex <= m_header.totalKeysCount) { std::string token; BinaryWalkerADL::Read(m_walker, token); - if (token.empty()) continue; + /** + * @brief We should save each available token. + * Empty tokens are valid too. + * Ref to ZTokenTable_Serializerlib::Load + */ m_tokenTable.emplace_back(std::move(token)); ++tokenIndex; @@ -275,7 +279,6 @@ namespace ReGlacier case TAG_NamedReference: case TAG_Reference: spdlog::info("[OP| +{:0X}] Reference (opcode {:2X})", offset, byte); - assert(false); break; case TAG_Char: case TAG_NamedChar: @@ -514,22 +517,10 @@ namespace ReGlacier auto geoms = gms->GetGeoms(); PRPWalkController controller {}; - for (const auto& currentGeom : geoms) - { - spdlog::info("Processing properties for geom {} ({:0X})", currentGeom.groupName, currentGeom.id); - do - { - controller.OnByte(this, m_walker.ReadUInt8()); - } - while ((controller.IsInContainer() || controller.IsInObject() || controller.IsInArray()) && !controller.IsEndOfStream() && !controller.IsError()); - + do { assert(!controller.IsError()); - - if (controller.IsEndOfStream()) - break; - - controller.Reset(); - } + controller.OnByte(this, m_walker.ReadUInt8()); + } while (!controller.IsEndOfStream()); return true; } @@ -614,13 +605,28 @@ namespace ReGlacier void IPRPVisitor::Visit_String(PRPWalker* walker, PRPToken token, std::string& result) { - const uint32_t index = walker->m_walker.ReadUInt32(); - if (index > walker->m_tokenTable.size()) { - spdlog::error("IPRPVisitor::Visit_String| Index of string is out of bounds! Index is {:8X}", index); - return; + if (((walker->GetHeaderBFlags() >> 3) & 1) == 0) { + uint32_t length { 0 }; + Visit_I32(walker, token, length); + + assert(length > 0); + + result.resize(length + 1); + result[length] = 0x0; + + Visit_RawContents(walker, token, result.data(), length); } + else + { + const uint32_t index = walker->m_walker.ReadUInt32(); + if (index >= walker->m_tokenTable.size()) { + spdlog::error("IPRPVisitor::Visit_String| Index of string is out of bounds! Index is {:8X}", index); + assert(false); + return; + } - result = walker->m_tokenTable[index]; + result = walker->m_tokenTable[index]; + } } void IPRPVisitor::Visit_RawBuffer(PRPWalker* walker, @@ -638,6 +644,12 @@ namespace ReGlacier } } + void IPRPVisitor::Visit_RawContents(PRPWalker* walker, PRPToken token, char* buffer, size_t bufferSize) + { + assert(bufferSize > 0); + walker->m_walker.ReadArray(buffer, bufferSize); + } + void IPRPVisitor::Visit_EndOfStream(PRPWalker* walker) { walker->m_flags |= PRPWalker::Flags::END_OF_STREAM; From 53f6d62aae320d1db4a3d76917f89199adf02ee6 Mon Sep 17 00:00:00 2001 From: DronCode Date: Sun, 31 Jan 2021 15:23:38 +0300 Subject: [PATCH 4/7] GMSTool: * Removed wrong arch code (GMS loading inside PRP, it's a mistake) * Removed some wrong logs --- Tools/GMSInfo/include/PRP/PRPVFSM.h | 7 ++----- Tools/GMSInfo/source/PRP/PRP.cpp | 2 +- Tools/GMSInfo/source/PRP/PRPVFSM.cpp | 19 +++---------------- 3 files changed, 6 insertions(+), 22 deletions(-) diff --git a/Tools/GMSInfo/include/PRP/PRPVFSM.h b/Tools/GMSInfo/include/PRP/PRPVFSM.h index 4133dac..8caa597 100644 --- a/Tools/GMSInfo/include/PRP/PRPVFSM.h +++ b/Tools/GMSInfo/include/PRP/PRPVFSM.h @@ -16,9 +16,6 @@ namespace ReGlacier { - class LevelContainer; - class LevelAssets; - class IPRPVisitor; using PRPToken = uint32_t; @@ -85,11 +82,11 @@ namespace ReGlacier [[nodiscard]] bool HasErrors() const { return m_flags & Flags::ERROR_IN_STREAM; } [[nodiscard]] uint32_t GetHeaderBFlags() const { return m_header.bFlags; } - void Prepare(IPRPVisitor* visitor, LevelContainer* pLevel, LevelAssets* pAssets); + void Prepare(IPRPVisitor* visitor); private: bool LoadZDefines(IPRPVisitor* visitor); - bool LoadProperties(IPRPVisitor* visitor, LevelContainer* pLevel, LevelAssets* pAssets); + bool LoadProperties(IPRPVisitor* visitor); private: uint8_t GetCurrentByte() const; diff --git a/Tools/GMSInfo/source/PRP/PRP.cpp b/Tools/GMSInfo/source/PRP/PRP.cpp index a3ffd1d..8740dc4 100644 --- a/Tools/GMSInfo/source/PRP/PRP.cpp +++ b/Tools/GMSInfo/source/PRP/PRP.cpp @@ -119,7 +119,7 @@ namespace ReGlacier IPRPVisitor visitor {}; PRPWalker walker { std::move(prpBuffer), prpBufferSize }; - walker.Prepare(&visitor, m_container, m_assets); + walker.Prepare(&visitor); // BinaryWalker binaryWalker { prpBuffer.get(), prpBufferSize }; // diff --git a/Tools/GMSInfo/source/PRP/PRPVFSM.cpp b/Tools/GMSInfo/source/PRP/PRPVFSM.cpp index 0bfcdb9..3dba3a4 100644 --- a/Tools/GMSInfo/source/PRP/PRPVFSM.cpp +++ b/Tools/GMSInfo/source/PRP/PRPVFSM.cpp @@ -46,7 +46,7 @@ namespace ReGlacier { } - void PRPWalker::Prepare(IPRPVisitor* visitor, LevelContainer* pLevelContainer, LevelAssets* pAssets) + void PRPWalker::Prepare(IPRPVisitor* visitor) { if (!visitor) { assert(visitor != nullptr); @@ -101,8 +101,7 @@ namespace ReGlacier m_zDefines.clear(); LoadZDefines(visitor); - LoadProperties(visitor, pLevelContainer, pAssets); - spdlog::info("DBG"); + LoadProperties(visitor); } bool PRPWalker::LoadZDefines(IPRPVisitor* visitor) @@ -501,20 +500,8 @@ namespace ReGlacier } }; - bool PRPWalker::LoadProperties(IPRPVisitor* visitor, LevelContainer* pLevel, LevelAssets* pAssets) + bool PRPWalker::LoadProperties(IPRPVisitor* visitor) { - /** - * @note this is test code, please refactor this - * @todo REFACTOR THIS LATER - */ - auto gms = std::make_unique(pAssets->GMS, pLevel, pAssets); - if (!gms->Load()) - { - spdlog::error("PRPWalker| Failed to load properties (unable to load GMS)"); - return false; - } - - auto geoms = gms->GetGeoms(); PRPWalkController controller {}; do { From 7935e6a3e45b3b85d11c3830b768188572ae3ac3 Mon Sep 17 00:00:00 2001 From: DronCode Date: Sun, 31 Jan 2021 22:08:09 +0300 Subject: [PATCH 5/7] GMSTool: * Totally refactored version. Need to write exporter & importer. --- Tools/GMSInfo/include/PRP/DebugPRPVisitor.h | 32 + Tools/GMSInfo/include/PRP/IPRPVisitor.h | 35 + Tools/GMSInfo/include/PRP/PRP.h | 17 +- Tools/GMSInfo/include/PRP/PRPADL.h | 29 + Tools/GMSInfo/include/PRP/PRPTreeNode.h | 47 -- Tools/GMSInfo/include/PRP/PRPTypes.h | 39 +- Tools/GMSInfo/include/PRP/PRPVFSM.h | 131 ---- Tools/GMSInfo/include/PRP/PRPWalker.h | 105 +++ Tools/GMSInfo/include/PRP/PRP_ETag_Helpers.h | 15 + Tools/GMSInfo/source/PRP/DebugPRPVisitor.cpp | 77 ++ Tools/GMSInfo/source/PRP/PRP.cpp | 229 +----- Tools/GMSInfo/source/PRP/PRPTreeNode.cpp | 73 -- Tools/GMSInfo/source/PRP/PRPVFSM.cpp | 683 ------------------ Tools/GMSInfo/source/PRP/PRPWalker.cpp | 593 +++++++++++++++ Tools/GMSInfo/source/PRP/PRP_ETag_Helpers.cpp | 68 ++ 15 files changed, 987 insertions(+), 1186 deletions(-) create mode 100644 Tools/GMSInfo/include/PRP/DebugPRPVisitor.h create mode 100644 Tools/GMSInfo/include/PRP/IPRPVisitor.h create mode 100644 Tools/GMSInfo/include/PRP/PRPADL.h delete mode 100644 Tools/GMSInfo/include/PRP/PRPTreeNode.h delete mode 100644 Tools/GMSInfo/include/PRP/PRPVFSM.h create mode 100644 Tools/GMSInfo/include/PRP/PRPWalker.h create mode 100644 Tools/GMSInfo/include/PRP/PRP_ETag_Helpers.h create mode 100644 Tools/GMSInfo/source/PRP/DebugPRPVisitor.cpp delete mode 100644 Tools/GMSInfo/source/PRP/PRPTreeNode.cpp delete mode 100644 Tools/GMSInfo/source/PRP/PRPVFSM.cpp create mode 100644 Tools/GMSInfo/source/PRP/PRPWalker.cpp create mode 100644 Tools/GMSInfo/source/PRP/PRP_ETag_Helpers.cpp diff --git a/Tools/GMSInfo/include/PRP/DebugPRPVisitor.h b/Tools/GMSInfo/include/PRP/DebugPRPVisitor.h new file mode 100644 index 0000000..2fadf85 --- /dev/null +++ b/Tools/GMSInfo/include/PRP/DebugPRPVisitor.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace ReGlacier +{ + class DebugPRPVisitor final : public IPRPVisitor + { + public: + void Visit_BeginArray(uint32_t capacity) override; + void Visit_EndArray() override; + void Visit_BeginObject() override; + void Visit_EndObject() override; + void Visit_Container(uint32_t) override; + void Visit_Bitfield(uint32_t) override; + void Visit_Bool(bool value) override; + void Visit_Char(char ch) override; + void Visit_I8(uint8_t result) override; + void Visit_I16(uint16_t result) override; + void Visit_I32(uint32_t result) override; + void Visit_F32(float result) override; + void Visit_F64(double result) override; + void Visit_String(const std::string& result) override; + void Visit_RawBuffer(const std::shared_ptr& buffer, size_t& bufferSize) override; + void Visit_StringArray(const std::vector& stringArray) override; + void Visit_StringOrArray(const std::string& s) override; + void Visit_StringOrArray(uint32_t v) override; + void Visit_SkipMark() override; + void Visit_EndOfStream() override; + void Visit_UnknownTag(uint8_t byte) override; + }; +} \ No newline at end of file diff --git a/Tools/GMSInfo/include/PRP/IPRPVisitor.h b/Tools/GMSInfo/include/PRP/IPRPVisitor.h new file mode 100644 index 0000000..cce5165 --- /dev/null +++ b/Tools/GMSInfo/include/PRP/IPRPVisitor.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +namespace ReGlacier +{ + class IPRPVisitor + { + public: + virtual ~IPRPVisitor() = default; + + virtual void Visit_BeginArray(uint32_t capacity) = 0; + virtual void Visit_EndArray() = 0; + virtual void Visit_BeginObject() = 0; + virtual void Visit_EndObject() = 0; + virtual void Visit_Container(uint32_t) = 0; + virtual void Visit_Bitfield(uint32_t) = 0; + virtual void Visit_Bool(bool value) = 0; + virtual void Visit_Char(char ch) = 0; + virtual void Visit_I8(uint8_t result) = 0; + virtual void Visit_I16(uint16_t result) = 0; + virtual void Visit_I32(uint32_t result) = 0; + virtual void Visit_F32(float result) = 0; + virtual void Visit_F64(double result) = 0; + virtual void Visit_String(const std::string& result) = 0; + virtual void Visit_RawBuffer(const std::shared_ptr& buffer, size_t& bufferSize) = 0; + virtual void Visit_StringArray(const std::vector& stringArray) = 0; + virtual void Visit_StringOrArray(const std::string& s) = 0; + virtual void Visit_StringOrArray(uint32_t v) = 0; + virtual void Visit_SkipMark() = 0; + virtual void Visit_EndOfStream() = 0; + virtual void Visit_UnknownTag(uint8_t byte) = 0; + }; +} \ No newline at end of file diff --git a/Tools/GMSInfo/include/PRP/PRP.h b/Tools/GMSInfo/include/PRP/PRP.h index 74dfa5b..5989003 100644 --- a/Tools/GMSInfo/include/PRP/PRP.h +++ b/Tools/GMSInfo/include/PRP/PRP.h @@ -1,15 +1,17 @@ #pragma once #include -#include + +#include + +#include +#include #include #include namespace ReGlacier { - struct DictNode; - class PRP : public IGameEntity { public: @@ -20,14 +22,5 @@ namespace ReGlacier bool Load() override; void PrintInfo() override; - - private: - void TryToDecompileEntities(std::unique_ptr&& buffer, size_t bufferSize); - - private: - int m_keysCount { 0 }; - int m_valuesOffset { 0 }; - std::vector m_keys {}; - PRPTreeNode* m_tree { nullptr }; }; } \ No newline at end of file diff --git a/Tools/GMSInfo/include/PRP/PRPADL.h b/Tools/GMSInfo/include/PRP/PRPADL.h new file mode 100644 index 0000000..9a408c5 --- /dev/null +++ b/Tools/GMSInfo/include/PRP/PRPADL.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include +#include + +namespace ReGlacier +{ + template <> + struct BinaryWalkerADL + { + static void Read(const BinaryWalker& binaryWalker, SPRP& value) + { + binaryWalker.ReadArray(&value.magic[0], SPRP::kMagicSize); + value.bIsRaw = static_cast(binaryWalker.ReadUInt8()); + value.bFlags = binaryWalker.ReadUInt32(); + value.__Unknown__ = binaryWalker.ReadUInt32(); + value.totalKeysCount = binaryWalker.ReadUInt32(); + value.dataOffset = binaryWalker.ReadUInt32(); + } + + static void Write(BinaryWalker& binaryWalker, const SPRP& value) + { + spdlog::error("NOT IMPLEMENTED"); + assert(false); + } + }; +} \ No newline at end of file diff --git a/Tools/GMSInfo/include/PRP/PRPTreeNode.h b/Tools/GMSInfo/include/PRP/PRPTreeNode.h deleted file mode 100644 index 6ecaf5c..0000000 --- a/Tools/GMSInfo/include/PRP/PRPTreeNode.h +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -namespace ReGlacier -{ - using RawData = std::vector; - using PRPDataBlock = std::variant; - - class PRPTreeNode - { - private: - // Tree Control Block - PRPTreeNode* _parent {nullptr }; - std::vector _children {}; - - // Data - PRP_ETag _tag { PRP_ETag::NO_TAG }; - std::optional _inBufferOffset; - PRPDataBlock _dataBlock; - - public: - PRPTreeNode() = default; - explicit PRPTreeNode(PRP_ETag tag); - PRPTreeNode(PRPTreeNode* parent, PRP_ETag tag); - ~PRPTreeNode(); - - bool IsRoot() const; - void AddChild(PRPTreeNode* node); - void RemoveChild(PRPTreeNode* node); - [[nodiscard]] const std::vector& GetChildNodes() const; - PRPTreeNode* GetParent(); - - // Debug - [[nodiscard]] bool HasRefToBuffer() const; - [[nodiscard]] size_t GetRefToBuffer() const; - void SetRefToBuffer(size_t offset); - - // Data - [[nodiscard]] PRP_ETag GetTag() const; - void SetTag(PRP_ETag tag); - }; -} \ No newline at end of file diff --git a/Tools/GMSInfo/include/PRP/PRPTypes.h b/Tools/GMSInfo/include/PRP/PRPTypes.h index 51106c8..d813c11 100644 --- a/Tools/GMSInfo/include/PRP/PRPTypes.h +++ b/Tools/GMSInfo/include/PRP/PRPTypes.h @@ -19,6 +19,26 @@ namespace ReGlacier }; #pragma pack(pop) + using PRPToken = uint32_t; + + struct ZToken + { + static constexpr PRPToken Void = 0x0FFFFFFFF; + static constexpr PRPToken Unknown = 0x0FFFFFFFE; + static constexpr PRPToken Joker = 0x0FFFFFFFD; + }; + + enum EPropertyType : unsigned int + { + Type_14 = 0xE, + Type_10 = 0xA, + Type_9 = 0x9, + Type_8 = 0x8, + Type_7 = 0x7, + Type_2 = 0x2, + Type_1 = 0x1 + }; + enum PRP_ETag : uint8_t { TAG_Array = 0x1, @@ -34,6 +54,8 @@ namespace ReGlacier TAG_Float64 = 0xB, TAG_String = 0xC, TAG_RawData = 0xD, + TAG_StringOrArray_E = 0xE, + TAG_StringArray = 0xF, TAG_Bitfield = 0x10, TAG_EndArray = 0x7C, TAG_SkipMark = 0x7D, @@ -52,22 +74,9 @@ namespace ReGlacier TAG_NamedFloat64 = 0x8B, TAG_NamedString = 0x8C, TAG_NamedRawData = 0x8D, + TAG_StringOrArray_8E = 0x8E, TAG_NameBitfield = 0x8F, - TAG_UNKNOWN, //14,16-123,128,142 + TAG_UNKNOWN, //16-123,128,142 NO_TAG, - - /// My tags - TAG_StringOrArray_E = 0xE, - TAG_StringOrArray_8E = 0x8E, - - TAG_StringArray = 0xF, - }; - - struct PRP_ETag_Helpers - { - static PRP_ETag FromByte(uint8_t byte); - static std::string_view ToString(PRP_ETag tag); - static bool IsTagIncreaseDepthLevel(PRP_ETag tag); - static bool IsTagDecreateDepthLevel(PRP_ETag tag); }; } \ No newline at end of file diff --git a/Tools/GMSInfo/include/PRP/PRPVFSM.h b/Tools/GMSInfo/include/PRP/PRPVFSM.h deleted file mode 100644 index 8caa597..0000000 --- a/Tools/GMSInfo/include/PRP/PRPVFSM.h +++ /dev/null @@ -1,131 +0,0 @@ -/** - * @file PRPVFSM - * - * PRP File format Visitor FSM - */ -#pragma once - -#include -#include - -#include -#include -#include -#include -#include - -namespace ReGlacier -{ - class IPRPVisitor; - using PRPToken = uint32_t; - - struct ZToken - { - static constexpr PRPToken Void = 0x0FFFFFFFF; - static constexpr PRPToken Unknown = 0x0FFFFFFFE; - static constexpr PRPToken Joker = 0x0FFFFFFFD; - }; - - enum EPropertyType : unsigned int - { - Type_14 = 0xE, - Type_10 = 0xA, - Type_9 = 0x9, - Type_8 = 0x8, - Type_7 = 0x7, - Type_2 = 0x2, - Type_1 = 0x1 - }; - - class PRPWalker - { - friend class IPRPVisitor; - public: - enum Flags : unsigned int { - NO_FLAG = 0x0, - END_OF_STREAM = 1 << 1, - ERROR_IN_STREAM = 1 << 2 - }; - - using ZString = std::string; - using ZRefTab = std::vector; - using ZI32Vec = std::vector; - using ZF32Vec = std::vector; - - using ZDefine_t = std::variant< - bool, - uint8_t, - uint16_t, - uint32_t, - float, - double, - ZString, - ZI32Vec, - ZF32Vec, - ZRefTab>; - - private: - std::unique_ptr m_buffer; - unsigned int m_flags { Flags::NO_FLAG }; - size_t m_bufferSize { 0 }; - size_t m_keysCount { 0 }; - size_t m_valuesOffset { 0 }; - std::vector m_tokenTable {}; - std::map m_zDefines {}; - SPRP m_header { 0 }; - BinaryWalker m_walker {}; - - public: - PRPWalker(std::unique_ptr&& buffer, size_t bufferSize); - - [[nodiscard]] bool IsEOS() const { return m_flags & Flags::END_OF_STREAM; }; - [[nodiscard]] bool HasErrors() const { return m_flags & Flags::ERROR_IN_STREAM; } - [[nodiscard]] uint32_t GetHeaderBFlags() const { return m_header.bFlags; } - - void Prepare(IPRPVisitor* visitor); - - private: - bool LoadZDefines(IPRPVisitor* visitor); - bool LoadProperties(IPRPVisitor* visitor); - - private: - uint8_t GetCurrentByte() const; - - public: - size_t GetCurrentOffset() const; - }; - - class IPRPVisitor - { - public: - virtual ~IPRPVisitor() = default; - - /** - * @brief This is debug function - * @note DEBUG ONLY - */ - virtual void OnByte(PRPWalker* walker, uint8_t byte) {} - - virtual void ExchangeHeader(PRPWalker* walker, unsigned int type); - virtual void ExchangeFooter(PRPWalker* walker, unsigned int type); - virtual void ExchangeContainer(PRPWalker* walker, unsigned int* capacity); - - virtual void BeginArray(PRPWalker* walker, PRPToken token, unsigned int size); - virtual void EndArray(PRPWalker* walker); - - virtual void Visit_I8(PRPWalker* walker, PRPToken token, uint8_t& result); - virtual void Visit_I16(PRPWalker* walker, PRPToken token, uint16_t& result); - virtual void Visit_I32(PRPWalker* walker, PRPToken token, uint32_t& result); - virtual void Visit_F32(PRPWalker* walker, PRPToken token, float& result); - virtual void Visit_F64(PRPWalker* walker, PRPToken token, double& result); - virtual void Visit_String(PRPWalker* walker, PRPToken token, std::string& result); - virtual void Visit_RawBuffer(PRPWalker* walker, PRPToken token, std::shared_ptr& buffer, size_t& bufferSize); - virtual void Visit_RawContents(PRPWalker* walker, PRPToken token, char* buffer, size_t bufferSize); - - virtual void Visit_EndOfStream(PRPWalker* walker); - - template void ExchangeArray(PRPWalker* walker, PRPToken token, T* data, unsigned int size); - template <> void ExchangeArray(PRPWalker* walker, PRPToken token, uint32_t* data, unsigned int size); - template <> void ExchangeArray(PRPWalker* walker, PRPToken token, float* data, unsigned int size); - }; -} \ No newline at end of file diff --git a/Tools/GMSInfo/include/PRP/PRPWalker.h b/Tools/GMSInfo/include/PRP/PRPWalker.h new file mode 100644 index 0000000..5072743 --- /dev/null +++ b/Tools/GMSInfo/include/PRP/PRPWalker.h @@ -0,0 +1,105 @@ +/** + * @file PRPVFSM + * + * PRP File format Visitor FSM + */ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include + +namespace ReGlacier +{ + class IPRPVisitor; + + class PRPWalker + { + friend class IPRPVisitor; + public: + using ZString = std::string; + using ZRefTab = std::vector; + using ZI32Vec = std::vector; + using ZF32Vec = std::vector; + + using ZDefine_t = std::variant< + bool, + uint8_t, + uint16_t, + uint32_t, + float, + double, + ZString, + ZI32Vec, + ZF32Vec, + ZRefTab>; + + private: + std::unique_ptr m_buffer; + size_t m_bufferSize { 0 }; + + std::vector m_tokenTable {}; + std::map m_zDefines {}; + + SPRP m_header { 0 }; + BinaryWalker m_walker {}; + + enum ContextFlag : unsigned int + { + CF_NONE = 0, + CF_EOS = 1 << 0, + CF_ERROR = 1 << 1, + CF_OBJECT = 1 << 2, + CF_ARRAY = 1 << 3, + CF_CONTAINER = 1 << 4 + }; + + unsigned int m_contextFlags { CF_NONE }; + + public: + PRPWalker(std::unique_ptr&& buffer, size_t bufferSize); + + [[nodiscard]] uint32_t GetHeaderBFlags() const { return m_header.bFlags; } + + bool Prepare(IPRPVisitor* visitor); + + private: + bool ValidateHeader(); + bool LoadTokenTable(); + bool LoadZDefines(IPRPVisitor* visitor); + bool LoadProperties(IPRPVisitor* visitor); + void PrepareByte(IPRPVisitor* visitor, uint8_t byte); + + uint8_t ExchangeHeader(unsigned int type); + void ExchangeFooter(unsigned int type); + void ExchangeContainer(unsigned int* capacity); + + void BeginArray(unsigned int size); + void EndArray(); + + void Exchange_I8(uint8_t& result); + void Exchange_I16(uint16_t& result); + void Exchange_I32(uint32_t& result); + void Exchange_F32(float& result); + void Exchange_F64(double& result); + void Exchange_String(std::string& result); + void Exchange_RawBuffer(std::shared_ptr& buffer, size_t& bufferSize); + void Exchange_RawContents(char* buffer, size_t bufferSize); + + template void ExchangeArray(T* data, unsigned int size); + template <> void ExchangeArray(uint32_t* data, unsigned int size); + template <> void ExchangeArray(float* data, unsigned int size); + + private: + uint8_t GetCurrentByte() const; + + public: + size_t GetCurrentOffset() const; + }; +} \ No newline at end of file diff --git a/Tools/GMSInfo/include/PRP/PRP_ETag_Helpers.h b/Tools/GMSInfo/include/PRP/PRP_ETag_Helpers.h new file mode 100644 index 0000000..f74add6 --- /dev/null +++ b/Tools/GMSInfo/include/PRP/PRP_ETag_Helpers.h @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +#include + +namespace ReGlacier +{ + struct PRP_ETag_Helpers + { + static PRP_ETag FromByte(uint8_t byte); + static std::string_view ToString(PRP_ETag tag); + }; +} \ No newline at end of file diff --git a/Tools/GMSInfo/source/PRP/DebugPRPVisitor.cpp b/Tools/GMSInfo/source/PRP/DebugPRPVisitor.cpp new file mode 100644 index 0000000..8922cba --- /dev/null +++ b/Tools/GMSInfo/source/PRP/DebugPRPVisitor.cpp @@ -0,0 +1,77 @@ +#include + +#include + +#include +#include + +namespace ReGlacier +{ + namespace Helpers { + static std::string FormatByteArrayAsString(const uint8_t* buffer, size_t size, int align = 8) + { + std::stringstream ss; + + for (size_t i = 0; i < size; i++) + { + if (i == 0) { ss << '\t'; } + else if (i % align == 0) { ss << "\n\t"; } + else { + ss << std::hex << std::setfill('0') << std::setw(2) << static_cast(buffer[i]) << ' '; + } + } + + return ss.str(); + } + + static std::string JoinStringArray(const std::vector& arr, char ch = ',') { + std::string result; + + const size_t totalElements = arr.size(); + for (size_t i = 0; i < totalElements; i++) { + if (i + 1 < totalElements) { + result.append(fmt::format("{},", arr[i])); + } + else { + result.append(arr[i]); + } + } + + return result; + } + } + + void DebugPRPVisitor::Visit_BeginArray(uint32_t capacity) { spdlog::info("DebugPRPVisitor| Array({})", capacity); } + void DebugPRPVisitor::Visit_EndArray() { spdlog::info("DebugPRPVisitor| End Array"); } + void DebugPRPVisitor::Visit_BeginObject() { spdlog::info("DebugPRPVisitor| Begin Object"); } + void DebugPRPVisitor::Visit_EndObject() { spdlog::info("DebugPRPVisitor| End Object"); } + void DebugPRPVisitor::Visit_Container(uint32_t v) { spdlog::info("DebugPRPVisitor| Container ({})", v); } + void DebugPRPVisitor::Visit_Bitfield(uint32_t v) { spdlog::info("DebugPRPVisitor| Bitfield {}", v); } + void DebugPRPVisitor::Visit_Bool(bool v) { spdlog::info("DebugPRPVisitor| Bool {}", (v ? "True": "False")); } + void DebugPRPVisitor::Visit_Char(char v) { spdlog::info("DebugPRPVisitor| Char '{}'", v); } + void DebugPRPVisitor::Visit_I8(uint8_t v) { spdlog::info("DebugPRPVisitor| Int8 {}", v); } + void DebugPRPVisitor::Visit_I16(uint16_t v) { spdlog::info("DebugPRPVisitor| Int16 {}", v); } + void DebugPRPVisitor::Visit_I32(uint32_t v) { spdlog::info("DebugPRPVisitor| Int32 {}", v); } + void DebugPRPVisitor::Visit_F32(float v) { spdlog::info("DebugPRPVisitor| Float32 {:f}", v); } + void DebugPRPVisitor::Visit_F64(double v) { spdlog::info("DebugPRPVisitor| Float64 {}", v); } + void DebugPRPVisitor::Visit_String(const std::string& v) { spdlog::info("DebugPRPVisitor| String \"{}\"", v); } + void DebugPRPVisitor::Visit_RawBuffer(const std::shared_ptr& buffer, size_t& bufferSize) { + spdlog::info( + "DebugPRPVisitor| Raw Data ({} entities):\n{}", + bufferSize, + Helpers::FormatByteArrayAsString(buffer.get(), bufferSize) + ); + } + void DebugPRPVisitor::Visit_StringArray(const std::vector& stringArray) { + spdlog::info( + "DebugPRPVisitor| String Array ({} entities): {}", + stringArray.size(), + Helpers::JoinStringArray(stringArray) + ); + } + void DebugPRPVisitor::Visit_StringOrArray(const std::string& s) { spdlog::info("DebugPRPVisitor| StringOrArray (str): {}", s); } + void DebugPRPVisitor::Visit_StringOrArray(uint32_t v) { spdlog::info("DebugPRPVisitor| StringOrArray (i32): {}", v); } + void DebugPRPVisitor::Visit_SkipMark() { spdlog::info("DebugPRPVisitor| Skip mark"); } + void DebugPRPVisitor::Visit_EndOfStream() { spdlog::info("DebugPRPVisitor| End of stream"); } + void DebugPRPVisitor::Visit_UnknownTag(uint8_t v) { spdlog::warn("DebugPRPVisitor| Unknown tag {:2X}", v); } +} \ No newline at end of file diff --git a/Tools/GMSInfo/source/PRP/PRP.cpp b/Tools/GMSInfo/source/PRP/PRP.cpp index 8740dc4..fb58b95 100644 --- a/Tools/GMSInfo/source/PRP/PRP.cpp +++ b/Tools/GMSInfo/source/PRP/PRP.cpp @@ -1,6 +1,8 @@ #include #include -#include +#include +#include +#include #include #include @@ -12,99 +14,14 @@ #include -#include -#include -#include -#include - namespace ReGlacier { - static constexpr const char* kNoTag = "(NOT A TAG)"; - static constexpr const char* kExpectedIdentifier = "IOPacked v0.1"; - - static constexpr int kKeysOffset = 0x17; - static constexpr int kKeysListOffset = 0x1F; - - PRP_ETag PRP_ETag_Helpers::FromByte(uint8_t byte) - { - if (byte < PRP_ETag::TAG_Array || byte > PRP_ETag::TAG_NameBitfield) - { - return PRP_ETag::NO_TAG; - } - - if (byte == 0x0E) return PRP_ETag::TAG_StringOrArray_E; - if (byte == 0x8E) return PRP_ETag::TAG_StringOrArray_8E; - if (byte == 0x0F) return PRP_ETag::TAG_StringArray; - - if (byte == 128 || (byte >= 16 && byte <= 123)) - { - return PRP_ETag::TAG_UNKNOWN; - } - - return static_cast(byte); - } - - std::string_view PRP_ETag_Helpers::ToString(PRP_ETag tag) - { -#define RETAG(id) if (tag == (id)) { return #id; } - RETAG(PRP_ETag::TAG_Array); - RETAG(PRP_ETag::TAG_BeginObject); - RETAG(PRP_ETag::TAG_Reference); - RETAG(PRP_ETag::TAG_Container); - RETAG(PRP_ETag::TAG_Char); - RETAG(PRP_ETag::TAG_Bool); - RETAG(PRP_ETag::TAG_Int8); - RETAG(PRP_ETag::TAG_Int16); - RETAG(PRP_ETag::TAG_Int32); - RETAG(PRP_ETag::TAG_Float32); - RETAG(PRP_ETag::TAG_Float64); - RETAG(PRP_ETag::TAG_String); - RETAG(PRP_ETag::TAG_RawData); - RETAG(PRP_ETag::TAG_Bitfield); - RETAG(PRP_ETag::TAG_EndArray); - RETAG(PRP_ETag::TAG_SkipMark); - RETAG(PRP_ETag::TAG_EndObject); - RETAG(PRP_ETag::TAG_EndOfStream); - RETAG(PRP_ETag::TAG_NamedArray); - RETAG(PRP_ETag::TAG_BeginNamedObject); - RETAG(PRP_ETag::TAG_NamedReference); - RETAG(PRP_ETag::TAG_NamedContainer); - RETAG(PRP_ETag::TAG_NamedChar); - RETAG(PRP_ETag::TAG_NamedBool); - RETAG(PRP_ETag::TAG_NamedInt8); - RETAG(PRP_ETag::TAG_NamedInt16); - RETAG(PRP_ETag::TAG_NamedInt32); - RETAG(PRP_ETag::TAG_NamedFloat32); - RETAG(PRP_ETag::TAG_NamedFloat64); - RETAG(PRP_ETag::TAG_NamedString); - RETAG(PRP_ETag::TAG_NamedRawData); - RETAG(PRP_ETag::TAG_NameBitfield); - RETAG(PRP_ETag::TAG_UNKNOWN); - return kNoTag; -#undef RETAG - } - - bool PRP_ETag_Helpers::IsTagIncreaseDepthLevel(PRP_ETag tag) - { - return - tag == PRP_ETag::TAG_Array || tag == PRP_ETag::TAG_BeginObject || - tag == PRP_ETag::TAG_Container || tag == PRP_ETag::TAG_NamedContainer || - tag == PRP_ETag::TAG_BeginNamedObject || tag == PRP_ETag::TAG_NamedArray; - } - - bool PRP_ETag_Helpers::IsTagDecreateDepthLevel(PRP_ETag tag) - { - return tag == PRP_ETag::TAG_EndArray || tag == PRP_ETag::TAG_EndObject || tag == PRP_ETag::TAG_EndOfStream; - } - PRP::PRP(std::string name, LevelContainer* levelContainer, LevelAssets* levelAssets) : IGameEntity(name, levelContainer, levelAssets) {} PRP::~PRP() { - delete m_tree; - m_tree = nullptr; } bool PRP::Load() @@ -117,151 +34,13 @@ namespace ReGlacier return false; } - IPRPVisitor visitor {}; + DebugPRPVisitor visitor {}; PRPWalker walker { std::move(prpBuffer), prpBufferSize }; walker.Prepare(&visitor); - -// BinaryWalker binaryWalker { prpBuffer.get(), prpBufferSize }; -// -// { -// std::string headerStr = binaryWalker.ReadZString(strlen(kExpectedIdentifier)); -// -// if (std::string_view(headerStr) != std::string_view(kExpectedIdentifier)) -// { -// spdlog::error("PRP::Load| Failed to load PRP {}. Reason: bad header!", m_name); -// return false; -// } -// -// spdlog::info("PRP::Load| Header is OK"); -// binaryWalker.Seek(kKeysOffset, IBaseStreamWalker::BEGIN); -// } -// -// { -// m_keysCount = binaryWalker.ReadUInt32(); -// m_valuesOffset = binaryWalker.ReadUInt32(); -// -// if (m_valuesOffset >= prpBufferSize) -// { -// spdlog::error("PRP::Load| Failed to load PRP {}. Reason: Bad 'values' offset!", m_name); -// m_keysCount = 0; -// m_valuesOffset = 0; -// return false; -// } -// } -// -// spdlog::info("PRP::Load| Header looks OK."); -// spdlog::info("PRP::Load| Total keys: {}", m_keysCount); -// -// m_keys.clear(); -// m_keys.reserve(m_keysCount); -// -// while (m_keys.size() != m_keysCount) -// { -// std::string val = binaryWalker.ReadZString(); -// if (!val.empty()) -// { -// m_keys.push_back(val); -// } -// } -// -// // Decompile it -// TryToDecompileEntities(std::move(prpBuffer), prpBufferSize); - return true; } void PRP::PrintInfo() { -// spdlog::info("PRP Info: "); -// spdlog::info("Keys: {}", m_keysCount); -// int index = 0; -// for (index = 0; index < m_keysCount; index++) -// { -// spdlog::info(" [{}] = {}", index, m_keys[index]); -// } -// spdlog::info(" --- END OF PRP --- "); - } - - void PRP::TryToDecompileEntities(std::unique_ptr&& buffer, size_t bufferSize) - { -// BinaryWalker binaryWalker { buffer.get(), bufferSize }; -// binaryWalker.Seek(kKeysListOffset, IBaseStreamWalker::BEGIN); -// binaryWalker.Seek(m_valuesOffset, IBaseStreamWalker::CURR); -// spdlog::info("Current pos: {}", binaryWalker.GetPosition()); - -// // Iterate over bytes, try to find TAG, if tag occured - decompress it's data if we can -// PRP_ETag tag = PRP_ETag::NO_TAG; -// int tagId = 0; -// -// if (m_tree) -// { -// delete m_tree; -// m_tree = nullptr; -// } -// -// m_tree = new PRPTreeNode(nullptr, PRP_ETag::NO_TAG); // No tag at root -// auto currentNode = m_tree; -// -// do -// { -// auto pos = binaryWalker.GetPosition(); -// uint8_t byte = binaryWalker.ReadUInt8(); -// tag = PRP_ETag_Helpers::FromByte(byte); -// -// if (tag == PRP_ETag::NO_TAG) continue; -// const auto tagName = PRP_ETag_Helpers::ToString(tag); -// -// // Recognize tag flags -// if (PRP_ETag_Helpers::IsTagIncreaseDepthLevel(tag)) -// { -// // Allocate new depth level, insert as child, set current not to new depth controller -// auto newChild = new PRPTreeNode(currentNode, tag); -// currentNode->AddChild(newChild); -// newChild->SetRefToBuffer(pos); -// currentNode = newChild; -// } -// else if (PRP_ETag_Helpers::IsTagDecreateDepthLevel(tag)) -// { -// // Change depth level of tree - jump to parent node -// if (!currentNode->IsRoot()) -// { -// auto lastChild = new PRPTreeNode(currentNode, tag); -// lastChild->SetRefToBuffer(pos); -// currentNode->AddChild(lastChild); -// currentNode = currentNode->GetParent(); -// } -// else -// { -// spdlog::warn(" *** TREE TRAVELING WARNING: TAG {} AT +{:X} TOLD US THAT WE NEED TO JUMP UP, BUT NO WAY TO DO THIS! *** ", PRP_ETag_Helpers::ToString(tag), pos); -// } -// } -// else -// { -// // Normal tag, insert as child -// auto child = new PRPTreeNode(currentNode, tag); -// child->SetRefToBuffer(pos); -// currentNode->AddChild(child); -// } -// -// if (tag == PRP_ETag::TAG_Float32) -// { -// auto fvl = binaryWalker.Read(); -// spdlog::info("[{}] +{:X} {} = {}", tagId, pos, tagName, fvl); -// } -// else if (tag == PRP_ETag::TAG_Int32) -// { -// auto i32 = binaryWalker.Read(); -// spdlog::info("[{}] +{:X} {} = {} (hex 0x{:08X})", tagId, pos, tagName, i32, i32); -// } -// else -// { -// spdlog::info("[{}] +{:X} {} = ?", tagId, pos, tagName); -// } -// -// ++tagId; -// } -// while (tag != PRP_ETag::TAG_EndOfStream); -// -// spdlog::info("Total ETags: {}, Total Keys: {}", tagId + 1, m_keysCount); } } diff --git a/Tools/GMSInfo/source/PRP/PRPTreeNode.cpp b/Tools/GMSInfo/source/PRP/PRPTreeNode.cpp deleted file mode 100644 index ff41966..0000000 --- a/Tools/GMSInfo/source/PRP/PRPTreeNode.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include - -namespace ReGlacier -{ - PRPTreeNode::PRPTreeNode(PRP_ETag tag) : _tag(tag) {} - - PRPTreeNode::PRPTreeNode(PRPTreeNode* parent, PRP_ETag tag) : _parent(parent), _tag(tag) {} - - PRPTreeNode::~PRPTreeNode() - { - for (const auto& child : _children) - { - delete child; - } - _children.clear(); - } - - bool PRPTreeNode::IsRoot() const { return _parent == nullptr; } - - void PRPTreeNode::AddChild(PRPTreeNode* node) - { - if (!node) return; - node->_parent = this; - _children.push_back(node); - } - - void PRPTreeNode::RemoveChild(PRPTreeNode* node) - { - if (!node) return; - if (node->_parent != this) return; - node->_parent = nullptr; - auto it = std::find(std::begin(_children), std::end(_children), node); - if (it != std::end(_children)) - { - _children.erase(it); - } - } - - const std::vector& PRPTreeNode::GetChildNodes() const - { - return _children; - } - - PRPTreeNode * PRPTreeNode::GetParent() - { - return _parent; - } - - bool PRPTreeNode::HasRefToBuffer() const - { - return _inBufferOffset.has_value(); - } - - size_t PRPTreeNode::GetRefToBuffer() const - { - return _inBufferOffset.has_value() ? _inBufferOffset.value() : 0; - } - - void PRPTreeNode::SetRefToBuffer(size_t offset) - { - _inBufferOffset = offset; - } - - PRP_ETag PRPTreeNode::GetTag() const - { - return _tag; - } - - void PRPTreeNode::SetTag(PRP_ETag tag) - { - _tag = tag; - } -} \ No newline at end of file diff --git a/Tools/GMSInfo/source/PRP/PRPVFSM.cpp b/Tools/GMSInfo/source/PRP/PRPVFSM.cpp deleted file mode 100644 index 3dba3a4..0000000 --- a/Tools/GMSInfo/source/PRP/PRPVFSM.cpp +++ /dev/null @@ -1,683 +0,0 @@ -#include -#include -#include -#include - -#include - -#include -#include - -#include - -#include -#include -#include - -namespace ReGlacier -{ - template <> - struct BinaryWalkerADL - { - static void Read(const BinaryWalker& binaryWalker, SPRP& value) - { - binaryWalker.ReadArray(&value.magic[0], SPRP::kMagicSize); - value.bIsRaw = static_cast(binaryWalker.ReadUInt8()); - value.bFlags = binaryWalker.ReadUInt32(); - value.__Unknown__ = binaryWalker.ReadUInt32(); - value.totalKeysCount = binaryWalker.ReadUInt32(); - value.dataOffset = binaryWalker.ReadUInt32(); - } - - static void Write(BinaryWalker& binaryWalker, const SPRP& value) - { - spdlog::error("NOT IMPLEMENTED"); - assert(false); - } - }; - - static constexpr const char* kExpectedIdentifier = "IOPacked v0.1"; - static constexpr int kKeysOffset = 0x17; - - PRPWalker::PRPWalker(std::unique_ptr&& buffer, size_t bufferSize) - : m_buffer(std::move(buffer)) - , m_bufferSize(bufferSize) - , m_flags(Flags::NO_FLAG) - { - } - - void PRPWalker::Prepare(IPRPVisitor* visitor) - { - if (!visitor) { - assert(visitor != nullptr); - return; - } - - m_walker = BinaryWalker { m_buffer.get(), m_bufferSize }; - BinaryWalkerADL::Read(m_walker, m_header); - - if (std::string_view { m_header.magic } != std::string_view { kExpectedIdentifier }) { - m_flags |= Flags::ERROR_IN_STREAM; - spdlog::error("PRPWalker| Bad PRP file (magic mismatch!)"); - return; - } - - if (m_header.totalKeysCount >= 0xDEADBEEF) { - m_flags |= Flags::ERROR_IN_STREAM; - spdlog::error("PRPWalker| Bad PRP file (too much tokens in table"); - return; - } - - if (m_header.dataOffset >= m_bufferSize) { - m_flags |= Flags::ERROR_IN_STREAM; - spdlog::error("PRPWalker| Bad PRP file (bad data offset"); - return; - } - - spdlog::info("PRP looks ok. Reading tokens table"); - m_tokenTable.clear(); - - m_tokenTable.reserve(m_header.totalKeysCount); - - int tokenIndex = 0; - while (tokenIndex <= m_header.totalKeysCount) - { - std::string token; - BinaryWalkerADL::Read(m_walker, token); - /** - * @brief We should save each available token. - * Empty tokens are valid too. - * Ref to ZTokenTable_Serializerlib::Load - */ - - m_tokenTable.emplace_back(std::move(token)); - ++tokenIndex; - } - - spdlog::info("Tokens table is ready. Total tokens count is {}", m_header.totalKeysCount); - m_walker.Seek(sizeof(SPRP) + m_header.dataOffset, BinaryWalker::BEGIN); //Reset position - m_walker.Seek(4, BinaryWalker::CURR); //HACK! TODO: Find a way to fix this - - m_zDefines.clear(); - - LoadZDefines(visitor); - LoadProperties(visitor); - } - - bool PRPWalker::LoadZDefines(IPRPVisitor* visitor) - { - // Load ZDefines - uint32_t capacity { 0 }; - - visitor->ExchangeContainer(this, &capacity); - - for (int i = 0; i < capacity; i++) - { - std::string name; - uint32_t typeId { 0 }; - - /// Get Name - visitor->ExchangeHeader(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); - visitor->Visit_String(this, ZToken::Void, name); - visitor->ExchangeFooter(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); - - /// Get Type ID - visitor->ExchangeHeader(this, EPropertyType::Type_8); - visitor->Visit_I32(this, ZToken::Void, typeId); - visitor->ExchangeFooter(this, EPropertyType::Type_8); - - ZDefine_t value; - - /// Read internal data - switch (typeId) - { - case 2: - { - uint32_t size { 0 }; - - // Read Size - visitor->ExchangeHeader(this, EPropertyType::Type_8); - visitor->Visit_I32(this, ZToken::Void, size); - visitor->ExchangeFooter(this, EPropertyType::Type_8); - - // Read Data - ZI32Vec vec; - vec.resize(size); - visitor->ExchangeArray(this, ZToken::Void, vec.data(), size); - - value = vec; - } - break; - case 3: - { - uint32_t size { 0 }; - - // Read Size - visitor->ExchangeHeader(this, EPropertyType::Type_8); - visitor->Visit_I32(this, ZToken::Void, size); - visitor->ExchangeFooter(this, EPropertyType::Type_8); - - // Read Data - ZF32Vec vec; - vec.resize(size); - visitor->ExchangeArray(this, ZToken::Void, vec.data(), size); - - value = vec; - } - break; - case 0xC: - case 0xE: - case 0x10: - { - std::string geomRefStr; - - visitor->ExchangeHeader(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); - visitor->Visit_String(this, ZToken::Void, geomRefStr); - visitor->ExchangeFooter(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); - - value = geomRefStr; - } - break; - case 0x11: - { - unsigned int elementsNr { 0 }; - - visitor->ExchangeContainer(this, &elementsNr); - std::vector reftab; - reftab.reserve(elementsNr); - - for (int j = 0; j < elementsNr; j++) - { - std::string geomName; - visitor->ExchangeHeader(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); - visitor->Visit_String(this, ZToken::Void, geomName); - visitor->ExchangeFooter(this, EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); - - reftab.push_back(geomName); - } - - value = reftab; - } - break; - default: - spdlog::warn("Unknown type id {}", typeId); - assert(false); - break; - } - - m_zDefines[name] = value; - } - - return true; - } - - class PRPWalkController : public IPRPVisitor - { - enum ContextFlag : unsigned int - { - CF_NONE = 0, - CF_EOS = 1 << 0, - CF_ERROR = 1 << 1, - CF_OBJECT = 1 << 2, - CF_ARRAY = 1 << 3, - CF_CONTAINER = 1 << 4 - }; - - unsigned int m_contextFlags { CF_NONE }; - - public: - void OnByte(PRPWalker* walker, uint8_t byte) override - { - if (IsEndOfStream()) return; - - const size_t offset = walker->GetCurrentOffset() - 1; - const PRP_ETag tag = PRP_ETag_Helpers::FromByte(byte); - - - switch (tag) { - case TAG_Array: - case TAG_NamedArray: - { - m_contextFlags |= ContextFlag::CF_ARRAY; - - uint32_t capacity = 0; - - Visit_I32(walker, ZToken::Void, capacity); - assert(capacity > 0); - - spdlog::info("[OP| +{:0X}] Begin array (size {}, opcode {:2X})", offset, capacity, byte); - } - break; - case TAG_BeginObject: - case TAG_BeginNamedObject: - m_contextFlags |= ContextFlag::CF_OBJECT; - spdlog::info("[OP| +{:0X}] Begin object (opcode {:2X})", offset, byte); - break; - case TAG_Container: - case TAG_NamedContainer: - { - m_contextFlags |= ContextFlag::CF_CONTAINER; - uint32_t v = 0; - Visit_I32(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] Begin container <{}> (opcode {:2X})", offset, v, byte); - } - break; - case TAG_EndArray: - m_contextFlags &= ~ContextFlag::CF_ARRAY; - spdlog::info("[OP| +{:0X}] End array", offset); - break; - case TAG_EndObject: - m_contextFlags &= ~(ContextFlag::CF_OBJECT | ContextFlag::CF_CONTAINER); - spdlog::info("[OP| +{:0X}] End object", offset); - break; - case TAG_EndOfStream: - m_contextFlags |= ContextFlag::CF_EOS; - spdlog::info("[OP| +{:0X}] >>> END OF STREAM <<<", offset); - break; - - case TAG_NamedReference: - case TAG_Reference: - spdlog::info("[OP| +{:0X}] Reference (opcode {:2X})", offset, byte); - break; - case TAG_Char: - case TAG_NamedChar: - { - uint8_t v = 0; - Visit_I8(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] Char {:c} (opcode {:2x})", offset, v, byte); - } - break; - case TAG_Bool: - case TAG_NamedBool: - { - uint8_t v = 0; - Visit_I8(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] Bool {} (opcode {:2X})", offset, v, byte); - } - break; - case TAG_Int8: - case TAG_NamedInt8: - { - uint8_t v = 0; - Visit_I8(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] Int8 {} (opcode {:2X})", offset, v, byte); - } - break; - case TAG_Int16: - case TAG_NamedInt16: - { - uint16_t v = 0; - Visit_I16(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] Int16 {} (opcode {:2X})", offset, v, byte); - } - break; - case TAG_Int32: - case TAG_NamedInt32: - { - uint32_t v = 0; - Visit_I32(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] Int32 {} (opcode {:2X})", offset, v, byte); - } - break; - case TAG_Float32: - case TAG_NamedFloat32: - { - float v = .0f; - Visit_F32(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] Float32 {} (opcode {:2X})", offset, v, byte); - } - break; - case TAG_Float64: - case TAG_NamedFloat64: - { - double v = .0f; - Visit_F64(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] Float32 {} (opcode {:2X})", offset, v, byte); - } - break; - case TAG_String: - case TAG_NamedString: - { - std::string v = {}; - Visit_String(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] String: \"{}\" (opcode {:2X})", offset, v, byte); - } - break; - case TAG_RawData: - case TAG_NamedRawData: - { - size_t bufferSize { 0 }; - std::shared_ptr buffer; - - Visit_RawBuffer(walker, ZToken::Void, buffer, bufferSize); - - if (buffer && bufferSize) - { - spdlog::info("[OP| +{:0X}] TAG_RawData (opcode {:2X}). Buffer of size {:08X} bytes", offset, byte, bufferSize); - std::stringstream ss; - - for (size_t i = 0; i < bufferSize; i++) - { - if (i != 0 && i % 8 == 0) { - ss << '\n'; - } - else - { - ss << std::hex << std::setfill('0') << std::setw(2) << static_cast(buffer.get()[i]) << ' '; - } - } - spdlog::info("{}", ss.str()); - } - else - { - spdlog::error("[OP| +{:0X}] TAG_RawData (opcode {:2X}). Bad buffer!", offset, byte); - assert(false); - } - } - break; - case TAG_Bitfield: - case TAG_NameBitfield: - { - uint32_t v { 0 }; - Visit_I32(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] TAG_Bitfield: {} (opcode {:2X})", offset, v, byte); - } - break; - case TAG_SkipMark: - spdlog::info("[OP| +{:0X}] TAG_SkipMark. Just skip current opcode", offset); - break; - /// >>> MY TAGS <<< - case TAG_StringOrArray_E: - case TAG_StringOrArray_8E: - { - if ((walker->GetHeaderBFlags() >> 2) & 1) { - std::string s {}; - Visit_String(walker, ZToken::Void, s); - spdlog::info("[OP| +{:0X}] TAG_StringOrArray. Read as string: {} (opcode {:2X})", offset, s, byte); - } - else - { - uint32_t v { 0 }; - Visit_I32(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] TAG_StringOrArray. Read as array: {} (i32, opcode {:2X})", offset, v, byte); - } - } - break; - case TAG_StringArray: - { - if ((walker->GetHeaderBFlags() >> 2) & 1) - { - if ((walker->GetHeaderBFlags() >> 3) & 1) - { - //Impl #1 - uint32_t capacity { 0 }; - Visit_I32(walker, ZToken::Void, capacity); - - if (capacity == 0) { - spdlog::info("[OP| +{:X}] TAG_StringArray (impl 1, opcode {:2X}). Array of strings: (EMPTY)", offset, byte); - } - else { - spdlog::info("[OP| +{:X}] TAG_StringArray (impl 1, opcode {:2X}). Array of strings ({}): ", offset, byte, capacity); - - for (int i = 0; i < capacity; i++) - { - std::string s; - Visit_String(walker, ZToken::Void, s); - - spdlog::info("\t[{}] = \"{}\"", i, s); - } - } - } - else - { - //Impl #2 (at 0x458CD0) - spdlog::error("[OP| +{:X}] TAG_StringArray (impl 2, opcode {:2X}). Not implemented", offset, byte); - assert(false); -// uint32_t v1 { 0 }; -// Visit_I32(walker, ZToken::Void, v1); ///??????????? -// -// uint32_t capacity { 0 }; -// Visit_I32(walker, ZToken::Void, capacity); -// -// if (capacity > 0) -// { -// uint32_t v2 { 0 }; -// Visit_I32(walker, ZToken::Void, v2); -// -// } - } - } - else - { - uint32_t v { 0 }; - Visit_I32(walker, ZToken::Void, v); - spdlog::info("[OP| +{:0X}] TAG_StringArray (impl 3, opcode {:2X}). Array as I32: {}", offset, byte, v); - } - } - break; - /// ETC - case TAG_UNKNOWN: - { - uint32_t v = 0; - Visit_I32(walker, ZToken::Void, v); - spdlog::warn("[OP| +{:0X}] TAK_UNKNOWN. Read next I32 {} (opcode {:2X})", offset, v, byte); - } - break; - case NO_TAG: - default: - { - m_contextFlags |= ContextFlag::CF_ERROR; - spdlog::error("[OP| +{:X}] Unknown opcode {:2X}", offset, byte); - assert(false); - } - break; - } - } - - void Reset() - { - m_contextFlags = ContextFlag::CF_NONE; - } - - [[nodiscard]] bool IsInObject() const { - return m_contextFlags & ContextFlag::CF_OBJECT; - } - - [[nodiscard]] bool IsInArray() const { - return m_contextFlags & ContextFlag::CF_ARRAY; - } - - [[nodiscard]] bool IsInContainer() const { - return m_contextFlags & ContextFlag::CF_CONTAINER; - } - - [[nodiscard]] bool IsEndOfStream() const { - return m_contextFlags & ContextFlag::CF_EOS; - } - - [[nodiscard]] bool IsError() const { - return m_contextFlags & ContextFlag::CF_ERROR; - } - }; - - bool PRPWalker::LoadProperties(IPRPVisitor* visitor) - { - PRPWalkController controller {}; - - do { - assert(!controller.IsError()); - controller.OnByte(this, m_walker.ReadUInt8()); - } while (!controller.IsEndOfStream()); - - return true; - } - - uint8_t PRPWalker::GetCurrentByte() const { - return m_walker.ReadInt8(); - } - - size_t PRPWalker::GetCurrentOffset() const { - return m_walker.GetPosition(); - } - - /// --------------------------------------------------------------------------------------------------------- - - void IPRPVisitor::ExchangeHeader(PRPWalker* walker, unsigned int /*type*/) - { - walker->m_walker.ReadUInt8(); - } - - void IPRPVisitor::ExchangeFooter(PRPWalker* walker, unsigned int type) - { - if (type == EPropertyType::Type_14) { - if (walker->m_header.bFlags & 1) { - walker->m_walker.ReadUInt8(); - } - } - } - - void IPRPVisitor::ExchangeContainer(PRPWalker* walker, unsigned int* capacity) - { - ExchangeHeader(walker, EPropertyType::Type_10); - - const uint32_t res = walker->m_walker.ReadUInt32(); - if (capacity) { - *capacity = res; - } - } - - void IPRPVisitor::BeginArray(PRPWalker* walker, PRPToken token, unsigned int size) - { - if (!(walker->m_header.bFlags & 1)) - return; - - walker->m_walker.ReadUInt8(); - - if (walker->m_header.bFlags & 1) - { - walker->m_walker.ReadUInt32(); - //TODO: Maybe we should check size constrains here? (walker->m_walker.ReadUInt32() == size) - } - } - - void IPRPVisitor::EndArray(PRPWalker* walker) - { - walker->m_walker.ReadUInt8(); - } - - void IPRPVisitor::Visit_I8(PRPWalker* walker, PRPToken token, uint8_t& result) - { - result = walker->GetCurrentByte(); - } - - void IPRPVisitor::Visit_I16(PRPWalker* walker, PRPToken token, uint16_t& result) - { - result = walker->m_walker.ReadUInt16(); - } - - void IPRPVisitor::Visit_I32(PRPWalker* walker, PRPToken token, uint32_t& result) - { - result = walker->m_walker.ReadUInt32(); - } - - void IPRPVisitor::Visit_F32(PRPWalker* walker, PRPToken token, float& result) - { - result = walker->m_walker.Read(); - } - - void IPRPVisitor::Visit_F64(PRPWalker* walker, PRPToken token, double& result) - { - result = walker->m_walker.Read(); - } - - void IPRPVisitor::Visit_String(PRPWalker* walker, PRPToken token, std::string& result) - { - if (((walker->GetHeaderBFlags() >> 3) & 1) == 0) { - uint32_t length { 0 }; - Visit_I32(walker, token, length); - - assert(length > 0); - - result.resize(length + 1); - result[length] = 0x0; - - Visit_RawContents(walker, token, result.data(), length); - } - else - { - const uint32_t index = walker->m_walker.ReadUInt32(); - if (index >= walker->m_tokenTable.size()) { - spdlog::error("IPRPVisitor::Visit_String| Index of string is out of bounds! Index is {:8X}", index); - assert(false); - return; - } - - result = walker->m_tokenTable[index]; - } - } - - void IPRPVisitor::Visit_RawBuffer(PRPWalker* walker, - PRPToken token, - std::shared_ptr& buffer, - size_t& bufferSize) { - uint32_t size { 0 }; - - Visit_I32(walker, token, size); - if (size) { - bufferSize = size; - - buffer = std::make_shared(size); - walker->m_walker.ReadArray(buffer.get(), bufferSize); - } - } - - void IPRPVisitor::Visit_RawContents(PRPWalker* walker, PRPToken token, char* buffer, size_t bufferSize) - { - assert(bufferSize > 0); - walker->m_walker.ReadArray(buffer, bufferSize); - } - - void IPRPVisitor::Visit_EndOfStream(PRPWalker* walker) - { - walker->m_flags |= PRPWalker::Flags::END_OF_STREAM; - spdlog::info("IPRPVisitor| End of stream"); - } - - template<> - void IPRPVisitor::ExchangeArray(PRPWalker* walker, PRPToken token, uint32_t* data, unsigned int size) - { - BeginArray(walker, token, size); - - auto sz = size; - uint32_t* pCurrent = data; - do { - ExchangeHeader(walker, EPropertyType::Type_7); - Visit_I32(walker, token, reinterpret_cast(*pCurrent)); - ExchangeFooter(walker, EPropertyType::Type_7); - - ++pCurrent; - --sz; - } while (sz); - - EndArray(walker); - } - - template<> - void IPRPVisitor::ExchangeArray(PRPWalker* walker, PRPToken token, float* data, unsigned int size) - { - BeginArray(walker, token, size); - - auto sz = size; - float* pCurrent = data; - do { - ExchangeHeader(walker, EPropertyType::Type_7); - Visit_F32(walker, token, reinterpret_cast(*pCurrent)); - ExchangeFooter(walker, EPropertyType::Type_7); - - ++pCurrent; - --sz; - } while (sz); - - EndArray(walker); - } -} \ No newline at end of file diff --git a/Tools/GMSInfo/source/PRP/PRPWalker.cpp b/Tools/GMSInfo/source/PRP/PRPWalker.cpp new file mode 100644 index 0000000..b70560c --- /dev/null +++ b/Tools/GMSInfo/source/PRP/PRPWalker.cpp @@ -0,0 +1,593 @@ +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include + +namespace ReGlacier +{ + static constexpr const char* kExpectedIdentifier = "IOPacked v0.1"; + + PRPWalker::PRPWalker(std::unique_ptr&& buffer, size_t bufferSize) + : m_buffer(std::move(buffer)) + , m_bufferSize(bufferSize) + { + } + + bool PRPWalker::Prepare(IPRPVisitor* visitor) + { + if (!visitor) { + assert(visitor != nullptr); + return false; + } + + if (!ValidateHeader()) { + spdlog::error("PRPWalker| Invalid PRP file"); + assert(false); + return false; + } + + spdlog::info("PRPWalker| PRP looks ok. Reading tokens table"); + m_tokenTable.clear(); + + if (!LoadTokenTable()) + { + spdlog::error("PRPWalker| Failed to load token table!"); + assert(false); + return false; + } + + spdlog::info("PRPWalker| Tokens table is ready. Total tokens count is {}", m_header.totalKeysCount); + m_walker.Seek(sizeof(SPRP) + m_header.dataOffset, BinaryWalker::BEGIN); //Reset position + m_walker.Seek(4, BinaryWalker::CURR); //HACK! TODO: Find a way to fix this + + m_zDefines.clear(); + + if (!LoadZDefines(visitor)) { + spdlog::error("PRPWalker| Failed to load ZDefines section"); + assert(false); + return false; + } + + if (!LoadProperties(visitor)) { + spdlog::error("PRPWalker| Failed to load properties section"); + assert(false); + return false; + } + + return true; + } + + bool PRPWalker::ValidateHeader() + { + m_walker = BinaryWalker { m_buffer.get(), m_bufferSize }; + BinaryWalkerADL::Read(m_walker, m_header); + + if (std::string_view { m_header.magic } != std::string_view { kExpectedIdentifier }) { + m_contextFlags |= ContextFlag::CF_ERROR; + spdlog::error("PRPWalker| Bad PRP file (magic mismatch!)"); + return false; + } + + if (m_header.totalKeysCount >= 0xDEADBEEF) { + m_contextFlags |= ContextFlag::CF_ERROR; + spdlog::error("PRPWalker| Bad PRP file (too much tokens in table"); + return false; + } + + if (m_header.dataOffset >= m_bufferSize) { + m_contextFlags |= ContextFlag::CF_ERROR; + spdlog::error("PRPWalker| Bad PRP file (bad data offset"); + return false; + } + + return true; + } + + bool PRPWalker::LoadTokenTable() + { + m_tokenTable.reserve(m_header.totalKeysCount); + + int tokenIndex = 0; + while (tokenIndex <= m_header.totalKeysCount) + { + std::string token; + BinaryWalkerADL::Read(m_walker, token); + /** + * @brief We should save each available token. + * Empty tokens are valid too. + * Ref to ZTokenTable_Serializerlib::Load + */ + + m_tokenTable.emplace_back(std::move(token)); + ++tokenIndex; + } + + return true; + } + + bool PRPWalker::LoadZDefines(IPRPVisitor* visitor) + { + // Load ZDefines + uint32_t capacity { 0 }; + + ExchangeContainer(&capacity); + + for (int i = 0; i < capacity; i++) + { + std::string name; + uint32_t typeId { 0 }; + + /// Get Name + ExchangeHeader(EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + Exchange_String(name); + ExchangeFooter(EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + + /// Get Type ID + ExchangeHeader(EPropertyType::Type_8); + Exchange_I32(typeId); + ExchangeFooter(EPropertyType::Type_8); + + ZDefine_t value; + + /// Read internal data + switch (typeId) + { + case 2: + { + uint32_t size { 0 }; + + // Read Size + ExchangeHeader(EPropertyType::Type_8); + Exchange_I32(size); + ExchangeFooter(EPropertyType::Type_8); + + // Read Data + ZI32Vec vec; + vec.resize(size); + ExchangeArray(vec.data(), size); + + value = vec; + } + break; + case 3: + { + uint32_t size { 0 }; + + // Read Size + ExchangeHeader(EPropertyType::Type_8); + Exchange_I32(size); + ExchangeFooter(EPropertyType::Type_8); + + // Read Data + ZF32Vec vec; + vec.resize(size); + ExchangeArray(vec.data(), size); + + value = vec; + } + break; + case 0xC: + case 0xE: + case 0x10: + { + std::string geomRefStr; + + ExchangeHeader(EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + Exchange_String(geomRefStr); + ExchangeFooter(EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + + value = geomRefStr; + } + break; + case 0x11: + { + unsigned int elementsNr { 0 }; + + ExchangeContainer(&elementsNr); + std::vector reftab; + reftab.reserve(elementsNr); + + for (int j = 0; j < elementsNr; j++) + { + std::string geomName; + ExchangeHeader(EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + Exchange_String(geomName); + ExchangeFooter(EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); + + reftab.push_back(geomName); + } + + value = reftab; + } + break; + default: + spdlog::warn("Unknown type id {}", typeId); + assert(false); + break; + } + + m_zDefines[name] = value; + } + + return true; + } + + bool PRPWalker::LoadProperties(IPRPVisitor* visitor) + { + do { + PrepareByte(visitor, m_walker.ReadUInt8()); + + if (m_contextFlags & ContextFlag::CF_ERROR) { + spdlog::error("PRPWalker| Got error on last prepare step. Check logs for details"); + assert(false); + return false; + } + + } while (!(m_contextFlags & ContextFlag::CF_EOS)); + + return true; + } + + void PRPWalker::PrepareByte(IPRPVisitor* visitor, uint8_t byte) + { + if (m_contextFlags & ContextFlag::CF_EOS) return; + + const size_t offset = GetCurrentOffset() - 1; + const PRP_ETag tag = PRP_ETag_Helpers::FromByte(byte); + + switch (tag) { + case TAG_Array: + case TAG_NamedArray: + { + m_contextFlags |= ContextFlag::CF_ARRAY; + + uint32_t capacity = 0; + + Exchange_I32(capacity); + if (capacity <= 0) { + m_contextFlags |= ContextFlag::CF_ERROR; + assert(capacity > 0); + return; + } + + visitor->Visit_BeginArray(capacity); + } + break; + case TAG_BeginObject: + case TAG_BeginNamedObject: + m_contextFlags |= ContextFlag::CF_OBJECT; + visitor->Visit_BeginObject(); + break; + case TAG_Container: + case TAG_NamedContainer: + { + m_contextFlags |= ContextFlag::CF_CONTAINER; + uint32_t v = 0; + Exchange_I32(v); + + visitor->Visit_Container(v); + } + break; + case TAG_EndArray: + m_contextFlags &= ~ContextFlag::CF_ARRAY; + visitor->Visit_EndArray(); + break; + case TAG_EndObject: + m_contextFlags &= ~(ContextFlag::CF_OBJECT | ContextFlag::CF_CONTAINER); + visitor->Visit_EndObject(); + break; + case TAG_EndOfStream: + m_contextFlags |= ContextFlag::CF_EOS; + visitor->Visit_EndOfStream(); + break; + case TAG_NamedReference: + case TAG_Reference: + //TODO: Visit reference + assert(false); + spdlog::error("[OP| +{:0X}] Reference (opcode {:2X}) not supported", offset, byte); + break; + case TAG_Char: + case TAG_NamedChar: + { + uint8_t v = 0; + Exchange_I8(v); + + visitor->Visit_Char(v); + } + break; + case TAG_Bool: + case TAG_NamedBool: + { + uint8_t v = 0; + Exchange_I8(v); + visitor->Visit_Bool(static_cast(v)); + } + break; + case TAG_Int8: + case TAG_NamedInt8: + { + uint8_t v = 0; + Exchange_I8(v); + visitor->Visit_I8(v); + } + break; + case TAG_Int16: + case TAG_NamedInt16: + { + uint16_t v = 0; + Exchange_I16(v); + visitor->Visit_I16(v); + } + break; + case TAG_Int32: + case TAG_NamedInt32: + { + uint32_t v = 0; + Exchange_I32(v); + visitor->Visit_I32(v); + } + break; + case TAG_Float32: + case TAG_NamedFloat32: + { + float v = .0f; + Exchange_F32(v); + visitor->Visit_F32(v); + } + break; + case TAG_Float64: + case TAG_NamedFloat64: + { + double v = .0f; + Exchange_F64(v); + visitor->Visit_F64(v); + } + break; + case TAG_String: + case TAG_NamedString: + { + std::string v = {}; + Exchange_String(v); + visitor->Visit_String(v); + } + break; + case TAG_RawData: + case TAG_NamedRawData: + { + size_t bufferSize { 0 }; + std::shared_ptr buffer; + + Exchange_RawBuffer(buffer, bufferSize); + + if (!bufferSize || !buffer) { + m_contextFlags |= ContextFlag::CF_ERROR; + spdlog::error("[OP| +{:0X}] TAG_RawData (opcode {:2X}). Bad buffer!", offset, byte); + assert(false); + return; + } + + visitor->Visit_RawBuffer(buffer, bufferSize); + } + break; + case TAG_Bitfield: + case TAG_NameBitfield: + { + uint32_t v { 0 }; + Exchange_I32(v); + visitor->Visit_Bitfield(v); + } + break; + case TAG_SkipMark: + visitor->Visit_SkipMark(); + break; + case TAG_StringOrArray_E: + case TAG_StringOrArray_8E: + { + if ((GetHeaderBFlags() >> 2) & 1) { + std::string s {}; + Exchange_String(s); + visitor->Visit_StringOrArray(s); + } + else + { + uint32_t v { 0 }; + Exchange_I32(v); + visitor->Visit_StringOrArray(v); + } + } + break; + case TAG_StringArray: + { + if ((GetHeaderBFlags() >> 2) & 1) + { + if ((GetHeaderBFlags() >> 3) & 1) + { + //Impl #1 + uint32_t capacity { 0 }; + Exchange_I32(capacity); + std::vector arr; + if (capacity) { arr.reserve(capacity); } + + for (int i = 0; i < capacity; i++) + { + std::string s; + Exchange_String(s); + arr.push_back(s); + } + + visitor->Visit_StringArray(arr); + } + else + { + //Impl #2 (at 0x458CD0) + spdlog::error("[OP| +{:X}] TAG_StringArray (impl 2, opcode {:2X}). Not implemented", offset, byte); + m_contextFlags |= ContextFlag::CF_ERROR; + assert(false); + } + } + else + { + uint32_t v { 0 }; + Exchange_I32(v); + visitor->Visit_I32(v); + } + } + break; + case TAG_UNKNOWN: + case NO_TAG: + default: + { + visitor->Visit_UnknownTag(byte); + m_contextFlags |= ContextFlag::CF_ERROR; + spdlog::error("[OP| +{:X}] Unknown opcode {:2X}", offset, byte); + assert(false); + } + break; + } + } + + uint8_t PRPWalker::ExchangeHeader(unsigned int /*type*/) { + return m_walker.ReadUInt8(); + } + + void PRPWalker::ExchangeFooter(unsigned int type) { + if (type == EPropertyType::Type_14) { + if (GetHeaderBFlags() & 1) { + m_walker.ReadUInt8(); + } + } + } + + void PRPWalker::ExchangeContainer(unsigned int* capacity) { + ExchangeHeader(EPropertyType::Type_10); + + const uint32_t res = m_walker.ReadUInt32(); + if (capacity) { + *capacity = res; + } + } + + void PRPWalker::BeginArray(unsigned int size) { + if (!(GetHeaderBFlags() & 1)) + return; + + m_walker.ReadUInt8(); + + if (GetHeaderBFlags() & 1) + { + m_walker.ReadUInt32(); + //TODO: Maybe we should check size constrains here? (walker->m_walker.ReadUInt32() == size) + } + } + + void PRPWalker::EndArray() { + m_walker.ReadUInt8(); + } + + void PRPWalker::Exchange_I8(uint8_t& result) { result = GetCurrentByte(); } + + void PRPWalker::Exchange_I16(uint16_t& result) { result = m_walker.ReadUInt16(); } + + void PRPWalker::Exchange_I32(uint32_t& result) { result = m_walker.ReadUInt32(); } + + void PRPWalker::Exchange_F32(float& result) { result = m_walker.Read(); } + + void PRPWalker::Exchange_F64(double& result) { result = m_walker.Read(); } + + void PRPWalker::Exchange_String(std::string& result) { + if (((GetHeaderBFlags() >> 3) & 1) == 0) { + uint32_t length { 0 }; + Exchange_I32(length); + + assert(length > 0); + + result.resize(length + 1); + result[length] = 0x0; + + Exchange_RawContents(result.data(), length); + } + else + { + const uint32_t index = m_walker.ReadUInt32(); + if (index >= m_tokenTable.size()) { + spdlog::error("IPRPVisitor::Visit_String| Index of string is out of bounds! Index is {:8X}", index); + assert(false); + return; + } + + result = m_tokenTable[index]; + } + } + + void PRPWalker::Exchange_RawBuffer(std::shared_ptr& buffer, size_t& bufferSize) { + uint32_t size { 0 }; + + Exchange_I32(size); + if (size) { + bufferSize = size; + + buffer = std::make_shared(size); + m_walker.ReadArray(buffer.get(), bufferSize); + } + } + + void PRPWalker::Exchange_RawContents(char* buffer, size_t bufferSize) { + assert(bufferSize > 0); + m_walker.ReadArray(buffer, bufferSize); + } + + template<> + void PRPWalker::ExchangeArray(uint32_t* data, unsigned int size) + { + BeginArray(size); + + auto sz = size; + uint32_t* pCurrent = data; + do { + ExchangeHeader(EPropertyType::Type_7); + Exchange_I32(reinterpret_cast(*pCurrent)); + ExchangeFooter(EPropertyType::Type_7); + + ++pCurrent; + --sz; + } while (sz); + + EndArray(); + } + + template<> + void PRPWalker::ExchangeArray(float* data, unsigned int size) + { + BeginArray(size); + + auto sz = size; + float* pCurrent = data; + do { + ExchangeHeader(EPropertyType::Type_7); + Exchange_F32(reinterpret_cast(*pCurrent)); + ExchangeFooter(EPropertyType::Type_7); + + ++pCurrent; + --sz; + } while (sz); + + EndArray(); + } + + uint8_t PRPWalker::GetCurrentByte() const { + return m_walker.ReadInt8(); + } + + size_t PRPWalker::GetCurrentOffset() const { + return m_walker.GetPosition(); + } +} \ No newline at end of file diff --git a/Tools/GMSInfo/source/PRP/PRP_ETag_Helpers.cpp b/Tools/GMSInfo/source/PRP/PRP_ETag_Helpers.cpp new file mode 100644 index 0000000..c385d16 --- /dev/null +++ b/Tools/GMSInfo/source/PRP/PRP_ETag_Helpers.cpp @@ -0,0 +1,68 @@ +#include + +namespace ReGlacier +{ + static constexpr const char* kNoTag = "(NOT A TAG)"; + + PRP_ETag PRP_ETag_Helpers::FromByte(uint8_t byte) + { + if (byte == 0x0E) return PRP_ETag::TAG_StringOrArray_E; + if (byte == 0x8E) return PRP_ETag::TAG_StringOrArray_8E; + if (byte == 0x0F) return PRP_ETag::TAG_StringArray; + + if (byte < PRP_ETag::TAG_Array || byte > PRP_ETag::TAG_NameBitfield) + { + return PRP_ETag::NO_TAG; + } + + if (byte == 128 || (byte >= 16 && byte <= 123)) + { + return PRP_ETag::TAG_UNKNOWN; + } + + return static_cast(byte); + } + + std::string_view PRP_ETag_Helpers::ToString(PRP_ETag tag) + { +#define RETAG(id) if (tag == (id)) { return #id; } + RETAG(PRP_ETag::TAG_Array); + RETAG(PRP_ETag::TAG_BeginObject); + RETAG(PRP_ETag::TAG_Reference); + RETAG(PRP_ETag::TAG_Container); + RETAG(PRP_ETag::TAG_Char); + RETAG(PRP_ETag::TAG_Bool); + RETAG(PRP_ETag::TAG_Int8); + RETAG(PRP_ETag::TAG_Int16); + RETAG(PRP_ETag::TAG_Int32); + RETAG(PRP_ETag::TAG_Float32); + RETAG(PRP_ETag::TAG_Float64); + RETAG(PRP_ETag::TAG_String); + RETAG(PRP_ETag::TAG_RawData); + RETAG(PRP_ETag::TAG_Bitfield); + RETAG(PRP_ETag::TAG_EndArray); + RETAG(PRP_ETag::TAG_SkipMark); + RETAG(PRP_ETag::TAG_EndObject); + RETAG(PRP_ETag::TAG_EndOfStream); + RETAG(PRP_ETag::TAG_NamedArray); + RETAG(PRP_ETag::TAG_BeginNamedObject); + RETAG(PRP_ETag::TAG_NamedReference); + RETAG(PRP_ETag::TAG_NamedContainer); + RETAG(PRP_ETag::TAG_NamedChar); + RETAG(PRP_ETag::TAG_NamedBool); + RETAG(PRP_ETag::TAG_NamedInt8); + RETAG(PRP_ETag::TAG_NamedInt16); + RETAG(PRP_ETag::TAG_NamedInt32); + RETAG(PRP_ETag::TAG_NamedFloat32); + RETAG(PRP_ETag::TAG_NamedFloat64); + RETAG(PRP_ETag::TAG_NamedString); + RETAG(PRP_ETag::TAG_NamedRawData); + RETAG(PRP_ETag::TAG_NameBitfield); + RETAG(PRP_ETag::TAG_UNKNOWN); + RETAG(PRP_ETag::TAG_StringArray); + RETAG(PRP_ETag::TAG_StringOrArray_E); + RETAG(PRP_ETag::TAG_StringOrArray_8E); + return kNoTag; +#undef RETAG + } +} \ No newline at end of file From 136bc11d1238636306f1d1aee263655d1e2aa601 Mon Sep 17 00:00:00 2001 From: DronCode Date: Mon, 1 Feb 2021 21:40:07 +0300 Subject: [PATCH 6/7] GMSTool: * Recognized unknown value after strings table --- Tools/GMSInfo/include/PRP/PRPWalker.h | 2 ++ Tools/GMSInfo/source/PRP/PRPWalker.cpp | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Tools/GMSInfo/include/PRP/PRPWalker.h b/Tools/GMSInfo/include/PRP/PRPWalker.h index 5072743..4cce8b5 100644 --- a/Tools/GMSInfo/include/PRP/PRPWalker.h +++ b/Tools/GMSInfo/include/PRP/PRPWalker.h @@ -49,6 +49,7 @@ namespace ReGlacier SPRP m_header { 0 }; BinaryWalker m_walker {}; + uint32_t m_totalObjectsPresented { 0 }; enum ContextFlag : unsigned int { @@ -66,6 +67,7 @@ namespace ReGlacier PRPWalker(std::unique_ptr&& buffer, size_t bufferSize); [[nodiscard]] uint32_t GetHeaderBFlags() const { return m_header.bFlags; } + [[nodiscard]] uint32_t GetTotalObjectsCount() const { return m_totalObjectsPresented; } bool Prepare(IPRPVisitor* visitor); diff --git a/Tools/GMSInfo/source/PRP/PRPWalker.cpp b/Tools/GMSInfo/source/PRP/PRPWalker.cpp index b70560c..d140f3c 100644 --- a/Tools/GMSInfo/source/PRP/PRPWalker.cpp +++ b/Tools/GMSInfo/source/PRP/PRPWalker.cpp @@ -47,7 +47,7 @@ namespace ReGlacier spdlog::info("PRPWalker| Tokens table is ready. Total tokens count is {}", m_header.totalKeysCount); m_walker.Seek(sizeof(SPRP) + m_header.dataOffset, BinaryWalker::BEGIN); //Reset position - m_walker.Seek(4, BinaryWalker::CURR); //HACK! TODO: Find a way to fix this + m_totalObjectsPresented = m_walker.ReadUInt32(); m_zDefines.clear(); @@ -221,6 +221,8 @@ namespace ReGlacier return true; } + static int g_totalObjectsCount { 0 }; + bool PRPWalker::LoadProperties(IPRPVisitor* visitor) { do { @@ -264,6 +266,7 @@ namespace ReGlacier break; case TAG_BeginObject: case TAG_BeginNamedObject: + ++g_totalObjectsCount; m_contextFlags |= ContextFlag::CF_OBJECT; visitor->Visit_BeginObject(); break; From a5f311bc84b069f4d3e0166a9c676c1398bfb33d Mon Sep 17 00:00:00 2001 From: DronCode Date: Mon, 1 Feb 2021 22:04:15 +0300 Subject: [PATCH 7/7] GMSTool: * Removed debug stuff, added more logs. PRP ZDefines section looks fine. --- Tools/GMSInfo/source/PRP/PRPWalker.cpp | 44 +++++++++++++++++++------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/Tools/GMSInfo/source/PRP/PRPWalker.cpp b/Tools/GMSInfo/source/PRP/PRPWalker.cpp index d140f3c..9ec6a69 100644 --- a/Tools/GMSInfo/source/PRP/PRPWalker.cpp +++ b/Tools/GMSInfo/source/PRP/PRPWalker.cpp @@ -136,12 +136,19 @@ namespace ReGlacier Exchange_I32(typeId); ExchangeFooter(EPropertyType::Type_8); - ZDefine_t value; + ZDefine_t value {}; + + constexpr uint32_t kTypeId_Array_I32 = 2; + constexpr uint32_t kTypeId_Array_F32 = 3; + constexpr uint32_t kTypeId_StringRef_Type1 = 0xC; + constexpr uint32_t kTypeId_StringRef_Type2 = 0xE; + constexpr uint32_t kTypeId_StringRef_Type3 = 0x10; + constexpr uint32_t kTypeId_StringREFTAB = 0x11; /// Read internal data switch (typeId) { - case 2: + case kTypeId_Array_I32: { uint32_t size { 0 }; @@ -156,9 +163,14 @@ namespace ReGlacier ExchangeArray(vec.data(), size); value = vec; + + spdlog::info("ZDefine| Array_I32 \"{}\" (length is {}):", name, size); + for (const auto& entry: vec) { + spdlog::info("ZDefine| Array_I32::Entry {}", entry); + } } break; - case 3: + case kTypeId_Array_F32: { uint32_t size { 0 }; @@ -173,22 +185,29 @@ namespace ReGlacier ExchangeArray(vec.data(), size); value = vec; + + spdlog::info("ZDefine| Array_F32 \"{}\" (length is {}):", name, size); + for (const auto& entry: vec) { + spdlog::info("ZDefine| Array_F32::Entry {:F}", entry); + } } break; - case 0xC: - case 0xE: - case 0x10: + case kTypeId_StringRef_Type1: + case kTypeId_StringRef_Type2: + case kTypeId_StringRef_Type3: { - std::string geomRefStr; + std::string geomRefStr {}; ExchangeHeader(EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); Exchange_String(geomRefStr); ExchangeFooter(EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); value = geomRefStr; + + spdlog::info("ZDefine| STRREF ({:X}) {} = \"{}\"", typeId, name, geomRefStr); } break; - case 0x11: + case kTypeId_StringREFTAB: { unsigned int elementsNr { 0 }; @@ -196,6 +215,8 @@ namespace ReGlacier std::vector reftab; reftab.reserve(elementsNr); + spdlog::info("ZDefine| REFTAB \"{}\" (length is {}):", name, elementsNr); + for (int j = 0; j < elementsNr; j++) { std::string geomName; @@ -204,13 +225,15 @@ namespace ReGlacier ExchangeFooter(EPropertyType::Type_2 | EPropertyType::Type_8 | EPropertyType::Type_1); reftab.push_back(geomName); + + spdlog::info("ZDefine| REFTAB::Entry \"{}\"", geomName); } value = reftab; } break; default: - spdlog::warn("Unknown type id {}", typeId); + spdlog::error("ZDefine| Unknown type id {}", typeId); assert(false); break; } @@ -221,8 +244,6 @@ namespace ReGlacier return true; } - static int g_totalObjectsCount { 0 }; - bool PRPWalker::LoadProperties(IPRPVisitor* visitor) { do { @@ -266,7 +287,6 @@ namespace ReGlacier break; case TAG_BeginObject: case TAG_BeginNamedObject: - ++g_totalObjectsCount; m_contextFlags |= ContextFlag::CF_OBJECT; visitor->Visit_BeginObject(); break;