diff --git a/Core/Libraries/CMakeLists.txt b/Core/Libraries/CMakeLists.txt index 1658bfbb9ee..206707f5ae0 100644 --- a/Core/Libraries/CMakeLists.txt +++ b/Core/Libraries/CMakeLists.txt @@ -10,3 +10,4 @@ add_subdirectory(Source/debug) add_subdirectory(Source/EABrowserDispatch) add_subdirectory(Source/EABrowserEngine) add_subdirectory(Source/Compression) +add_subdirectory(Source/DataChunk) diff --git a/Core/Libraries/Include/DataChunk/DataChunk.h b/Core/Libraries/Include/DataChunk/DataChunk.h new file mode 100644 index 00000000000..ec487c24812 --- /dev/null +++ b/Core/Libraries/Include/DataChunk/DataChunk.h @@ -0,0 +1,247 @@ +/* +** Command & Conquer Generals(tm) +** Copyright 2025 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +// DataChunk.h +// Main public API for DataChunk library +// TheSuperHackers @feature bobtista 14/11/2025 Extract chunk I/O to platform-neutral library + +#pragma once + +#include "DataChunk/Types.h" +#include "DataChunk/Stream.h" +#include "DataChunk/TableOfContents.h" +#include + +namespace DataChunk { + +//---------------------------------------------------------------------- +// OutputChunk +//---------------------------------------------------------------------- +/** Internal structure for tracking open output chunks. */ +struct OutputChunk +{ + OutputChunk* next; + ChunkUInt id; // chunk symbol type from table of contents + ChunkInt filepos; // position of file at start of data offset + + OutputChunk() : next(NULL), id(0), filepos(0) {} +}; + +//---------------------------------------------------------------------- +// InputChunk +//---------------------------------------------------------------------- +/** Internal structure for tracking open input chunks. */ +struct InputChunk +{ + InputChunk* next; + ChunkUInt id; // chunk symbol type from table of contents + DataChunkVersionType version; // version of data + ChunkInt chunkStart; // position of the start of chunk data (past header) + ChunkInt dataSize; // total data size of chunk + ChunkInt dataLeft; // data left to read in this chunk + + InputChunk() : next(NULL), id(0), version(0), chunkStart(0), dataSize(0), dataLeft(0) {} +}; + +//---------------------------------------------------------------------- +// DataChunkInfo +//---------------------------------------------------------------------- +/** Information about a chunk being parsed. */ +struct DataChunkInfo +{ + ChunkString label; + ChunkString parentLabel; + DataChunkVersionType version; + ChunkInt dataSize; +}; + +//---------------------------------------------------------------------- +// DataChunkParserPtr +//---------------------------------------------------------------------- +/** Function pointer type for parsing chunks. */ +typedef bool (*DataChunkParserPtr)(class DataChunkInput& file, DataChunkInfo* info, void* userData); + +//---------------------------------------------------------------------- +// UserParser +//---------------------------------------------------------------------- +/** Internal structure for registered parsers. */ +struct UserParser +{ + UserParser* next; + DataChunkParserPtr parser; // the user parsing function + ChunkString label; // the data chunk label to match + ChunkString parentLabel; // the parent chunk's label (the scope) + void* userData; // user data pointer + + UserParser() : next(NULL), parser(NULL), userData(NULL) {} +}; + +//---------------------------------------------------------------------- +// DataChunkOutput +//---------------------------------------------------------------------- +/** Class for writing chunk-based data files. + Platform-neutral replacement for engine's DataChunkOutput. */ +class DataChunkOutput +{ + DataChunkOutputStream* m_pOut; // The actual output stream + DataChunkTableOfContents m_contents; // table of contents of data chunk types + OutputChunk* m_chunkStack; // current stack of open data chunks + + // Internal buffer for writing (replaces temp file) + char* m_buffer; + unsigned int m_bufferSize; + unsigned int m_bufferPos; + + void growBuffer(unsigned int needed); + +public: + DataChunkOutput(DataChunkOutputStream* pOut); + ~DataChunkOutput(); + + /** Open a new data chunk. + @param name Chunk type name (will be added to string table) + @param ver Version number for this chunk */ + void openDataChunk(const char* name, DataChunkVersionType ver); + + /** Close the current data chunk. */ + void closeDataChunk(); + + /** Write a float value. */ + void writeReal(ChunkReal r); + + /** Write an integer value. */ + void writeInt(ChunkInt i); + + /** Write a byte value. */ + void writeByte(ChunkByte b); + + /** Write an ASCII string (length-prefixed, no null terminator). */ + void writeAsciiString(const ChunkString& string); + + /** Write a Unicode string (length-prefixed, no null terminator). */ + void writeUnicodeString(const ChunkWideString& string); + + /** Write an array of bytes. */ + void writeArrayOfBytes(const char* ptr, ChunkInt len); + + /** Allocate or get existing ID for a chunk type name. + @param name Chunk type name + @return Integer ID (existing or newly allocated) */ + ChunkUInt allocateID(const ChunkString& name); +}; + +//---------------------------------------------------------------------- +// DataChunkInput +//---------------------------------------------------------------------- +/** Class for reading chunk-based data files. + Platform-neutral replacement for engine's DataChunkInput. */ +class DataChunkInput +{ + enum { CHUNK_HEADER_BYTES = 4 }; // 2 shorts in chunk file header + + DataChunkInputStream* m_file; // input file stream + DataChunkTableOfContents m_contents; // table of contents of data chunk types + ChunkInt m_fileposOfFirstChunk; // seek position of first data chunk + UserParser* m_parserList; // list of all registered parsers + InputChunk* m_chunkStack; // current stack of open data chunks + + void clearChunkStack(); + void decrementDataLeft(ChunkInt size); + +public: + void* m_currentObject; // user parse routines can use this + void* m_userData; // user data hook + + DataChunkInput(DataChunkInputStream* pStream); + ~DataChunkInput(); + + /** Register a parser function for data chunks. + @param label Chunk label to match + @param parentLabel Parent chunk label (or empty for global scope) + @param parser Parser function to call + @param userData Optional user data to pass to parser */ + void registerParser(const ChunkString& label, const ChunkString& parentLabel, + DataChunkParserPtr parser, void* userData = NULL); + + /** Parse the chunk stream using registered parsers. + @param userData Optional user data to pass to parsers + @return true on success, false on failure */ + bool parse(void* userData = NULL); + + /** Check if file has valid chunk format. + @return true if valid format */ + bool isValidFileType(); + + /** Open the next data chunk. + @param ver Output parameter for chunk version + @return Chunk label name */ + ChunkString openDataChunk(DataChunkVersionType* ver); + + /** Close the current chunk and move to next. */ + void closeDataChunk(); + + /** Check if at end of file. + @return true if at end */ + bool atEndOfFile(); + + /** Check if at end of current chunk. + @return true if all data read from chunk */ + bool atEndOfChunk(); + + /** Reset to just-opened state. */ + void reset(); + + /** Get label of current chunk. + @return Chunk label name */ + ChunkString getChunkLabel(); + + /** Get version of current chunk. + @return Chunk version number */ + DataChunkVersionType getChunkVersion(); + + /** Get size of data in current chunk. + @return Data size in bytes */ + ChunkUInt getChunkDataSize(); + + /** Get size of data left to read in current chunk. + @return Remaining data size in bytes */ + ChunkUInt getChunkDataSizeLeft(); + + /** Read a float value. */ + ChunkReal readReal(); + + /** Read an integer value. */ + ChunkInt readInt(); + + /** Read a byte value. */ + ChunkByte readByte(); + + /** Read an ASCII string. */ + ChunkString readAsciiString(); + + /** Read a Unicode string. */ + ChunkWideString readUnicodeString(); + + /** Read an array of bytes. + @param ptr Buffer to read into + @param len Number of bytes to read */ + void readArrayOfBytes(char* ptr, ChunkInt len); +}; + +} // namespace DataChunk + diff --git a/Core/Libraries/Include/DataChunk/README.md b/Core/Libraries/Include/DataChunk/README.md new file mode 100644 index 00000000000..d4a0b7cc260 --- /dev/null +++ b/Core/Libraries/Include/DataChunk/README.md @@ -0,0 +1,105 @@ +# DataChunk Library + +Platform-neutral library for reading and writing chunk-based data files (SCB format). + +## Overview + +This library provides the core chunk I/O functionality extracted from the game engine, making it available for use in both the engine and standalone tools. It maintains 100% binary compatibility with retail SCB files. + +## Features + +- ✅ VC6 compatible (no C++11 features) +- ✅ Platform-neutral (works on Windows, macOS, Linux) +- ✅ Binary compatible with retail SCB format +- ✅ No engine dependencies +- ✅ Simple stream-based interface + +## Usage Example + +### Reading a Chunk File + +```cpp +#include "DataChunk/DataChunk.h" +#include "DataChunk/StreamAdapters.h" +#include + +using namespace DataChunk; + +bool parseScript(DataChunkInput& file, DataChunkInfo* info, void* userData) +{ + ChunkString scriptName = file.readAsciiString(); + ChunkString comment = file.readAsciiString(); + // ... read more fields ... + return true; +} + +int main() +{ + FILE* fp = fopen("script.scb", "rb"); + if (!fp) return 1; + + FileInputStream stream(fp); + DataChunkInput chunkInput(&stream); + + chunkInput.registerParser("Script", "", parseScript); + chunkInput.parse(); + + fclose(fp); + return 0; +} +``` + +### Writing a Chunk File + +```cpp +#include "DataChunk/DataChunk.h" +#include "DataChunk/StreamAdapters.h" +#include + +using namespace DataChunk; + +int main() +{ + FILE* fp = fopen("output.scb", "wb"); + if (!fp) return 1; + + FileOutputStream stream(fp); + DataChunkOutput chunkOutput(&stream); + + chunkOutput.openDataChunk("Script", 2); + chunkOutput.writeAsciiString("MyScript"); + chunkOutput.writeAsciiString("Comment"); + // ... write more fields ... + chunkOutput.closeDataChunk(); + + fclose(fp); + return 0; +} +``` + +## Architecture + +- **Stream.h**: Abstract stream interfaces +- **Types.h**: Type definitions (VC6 compatible) +- **TableOfContents.h/cpp**: String-to-ID mapping +- **DataChunk.h/cpp**: Main I/O classes +- **StreamAdapters.h**: FILE* adapters for tools + +## Binary Format + +The library maintains exact binary compatibility with the engine's chunk format: + +- Chunk header: 10 bytes (4-byte ID + 2-byte version + 4-byte size) +- String table: "CkMp" magic + count + entries +- Strings: Length-prefixed (2 bytes) + data (no null terminator) +- All integers: Little-endian, 4 bytes +- All floats: IEEE 754, 4 bytes + +## VC6 Compatibility Notes + +- Uses `std::string` (VC6 STL compatible) +- Uses `NULL` instead of `nullptr` +- No C++11 features +- Raw pointers (no smart pointers) +- Standard C++98/C++03 + diff --git a/Core/Libraries/Include/DataChunk/Stream.h b/Core/Libraries/Include/DataChunk/Stream.h new file mode 100644 index 00000000000..00ffa9a8a0c --- /dev/null +++ b/Core/Libraries/Include/DataChunk/Stream.h @@ -0,0 +1,77 @@ +/* +** Command & Conquer Generals(tm) +** Copyright 2025 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +// Stream.h +// Platform-neutral stream interfaces for DataChunk library +// TheSuperHackers @feature bobtista 14/11/2025 Extract chunk I/O to platform-neutral library + +#pragma once + +#include + +namespace DataChunk { + +//---------------------------------------------------------------------- +// DataChunkOutputStream +//---------------------------------------------------------------------- +/** Virtual interface for writing chunk data. Platform-neutral replacement + for engine's OutputStream. */ +class DataChunkOutputStream +{ +public: + virtual ~DataChunkOutputStream() {} + + /** Write data to the stream. + @param data Pointer to data to write + @param numBytes Number of bytes to write + @return Number of bytes written (should equal numBytes on success) */ + virtual unsigned int write(const void* data, unsigned int numBytes) = 0; +}; + +//---------------------------------------------------------------------- +// DataChunkInputStream +//---------------------------------------------------------------------- +/** Virtual interface for reading chunk data. Platform-neutral replacement + for engine's ChunkInputStream. */ +class DataChunkInputStream +{ +public: + virtual ~DataChunkInputStream() {} + + /** Read data from the stream. + @param data Buffer to read into + @param numBytes Number of bytes to read + @return Number of bytes read */ + virtual unsigned int read(void* data, unsigned int numBytes) = 0; + + /** Get current position in stream. + @return Current byte offset from start */ + virtual unsigned int tell() = 0; + + /** Seek to absolute position. + @param pos Byte offset from start + @return true on success, false on failure */ + virtual bool seek(unsigned int pos) = 0; + + /** Check if at end of stream. + @return true if at end, false otherwise */ + virtual bool eof() = 0; +}; + +} // namespace DataChunk + diff --git a/Core/Libraries/Include/DataChunk/StreamAdapters.h b/Core/Libraries/Include/DataChunk/StreamAdapters.h new file mode 100644 index 00000000000..dbe6f41feaa --- /dev/null +++ b/Core/Libraries/Include/DataChunk/StreamAdapters.h @@ -0,0 +1,102 @@ +/* +** Command & Conquer Generals(tm) +** Copyright 2025 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +// StreamAdapters.h +// FILE* stream adapters for DataChunk library (for use in tools) +// TheSuperHackers @feature bobtista 14/11/2025 Extract chunk I/O to platform-neutral library + +#pragma once + +#include "DataChunk/Stream.h" +#include + +namespace DataChunk { + +//---------------------------------------------------------------------- +// FileOutputStream +//---------------------------------------------------------------------- +/** Adapter that wraps a FILE* for writing chunk data. + Use this in standalone tools that work with FILE* streams. */ +class FileOutputStream : public DataChunkOutputStream +{ + FILE* m_file; + +public: + FileOutputStream(FILE* file) : m_file(file) {} + + virtual unsigned int write(const void* data, unsigned int numBytes) + { + if (m_file) + { + return (unsigned int)fwrite(data, 1, numBytes, m_file); + } + return 0; + } +}; + +//---------------------------------------------------------------------- +// FileInputStream +//---------------------------------------------------------------------- +/** Adapter that wraps a FILE* for reading chunk data. + Use this in standalone tools that work with FILE* streams. */ +class FileInputStream : public DataChunkInputStream +{ + FILE* m_file; + +public: + FileInputStream(FILE* file) : m_file(file) {} + + virtual unsigned int read(void* data, unsigned int numBytes) + { + if (m_file) + { + return (unsigned int)fread(data, 1, numBytes, m_file); + } + return 0; + } + + virtual unsigned int tell() + { + if (m_file) + { + return (unsigned int)ftell(m_file); + } + return 0; + } + + virtual bool seek(unsigned int pos) + { + if (m_file) + { + return fseek(m_file, (long)pos, SEEK_SET) == 0; + } + return false; + } + + virtual bool eof() + { + if (m_file) + { + return feof(m_file) != 0; + } + return true; + } +}; + +} // namespace DataChunk + diff --git a/Core/Libraries/Include/DataChunk/TableOfContents.h b/Core/Libraries/Include/DataChunk/TableOfContents.h new file mode 100644 index 00000000000..6efebc82269 --- /dev/null +++ b/Core/Libraries/Include/DataChunk/TableOfContents.h @@ -0,0 +1,92 @@ +/* +** Command & Conquer Generals(tm) +** Copyright 2025 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +// TableOfContents.h +// String-to-ID mapping table for DataChunk library +// TheSuperHackers @feature bobtista 14/11/2025 Extract chunk I/O to platform-neutral library + +#pragma once + +#include "DataChunk/Types.h" +#include "DataChunk/Stream.h" +#include + +namespace DataChunk { + +//---------------------------------------------------------------------- +// Mapping +//---------------------------------------------------------------------- +/** Internal structure for string-to-ID mapping. */ +struct Mapping +{ + Mapping* next; + ChunkString name; + ChunkUInt id; + + Mapping() : next(NULL), id(0) {} +}; + +//---------------------------------------------------------------------- +// DataChunkTableOfContents +//---------------------------------------------------------------------- +/** Manages the string table that maps chunk type names to integer IDs. + This is written at the start of chunk files and allows chunk types + to be identified by name rather than hardcoded IDs. */ +class DataChunkTableOfContents +{ + Mapping* m_list; + ChunkInt m_listLength; + ChunkUInt m_nextID; + bool m_headerOpened; + + Mapping* findMapping(const ChunkString& name); + +public: + DataChunkTableOfContents(); + ~DataChunkTableOfContents(); + + /** Get ID for a name (must exist). + @param name String name to look up + @return Integer ID for the name */ + ChunkUInt getID(const ChunkString& name); + + /** Get name for an ID (must exist). + @param id Integer ID to look up + @return String name for the ID */ + ChunkString getName(ChunkUInt id); + + /** Allocate or get existing ID for a name. + @param name String name + @return Integer ID (existing or newly allocated) */ + ChunkUInt allocateID(const ChunkString& name); + + /** Check if table was opened for reading. + @return true if table was read from file */ + bool isOpenedForRead() const { return m_headerOpened; } + + /** Write table to output stream. + @param out Output stream to write to */ + void write(DataChunkOutputStream& out); + + /** Read table from input stream. + @param in Input stream to read from */ + void read(DataChunkInputStream& in); +}; + +} // namespace DataChunk + diff --git a/Core/Libraries/Include/DataChunk/Types.h b/Core/Libraries/Include/DataChunk/Types.h new file mode 100644 index 00000000000..c5871fc60a1 --- /dev/null +++ b/Core/Libraries/Include/DataChunk/Types.h @@ -0,0 +1,45 @@ +/* +** Command & Conquer Generals(tm) +** Copyright 2025 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +// Types.h +// Type definitions for DataChunk library +// TheSuperHackers @feature bobtista 14/11/2025 Extract chunk I/O to platform-neutral library + +#pragma once + +#include +#include + +namespace DataChunk { + +// Type aliases matching engine types for binary compatibility +typedef uint16_t DataChunkVersionType; // matches engine's DataChunkVersionType + +// String types - using std::string for platform neutrality +typedef std::string ChunkString; +typedef std::wstring ChunkWideString; + +// Numeric types matching engine for binary compatibility +typedef int32_t ChunkInt; // matches engine's Int +typedef uint32_t ChunkUInt; // matches engine's UnsignedInt +typedef uint16_t ChunkUShort; // matches engine's UnsignedShort +typedef uint8_t ChunkByte; // matches engine's Byte +typedef float ChunkReal; // matches engine's Real + +} // namespace DataChunk + diff --git a/Core/Libraries/Source/DataChunk/CMakeLists.txt b/Core/Libraries/Source/DataChunk/CMakeLists.txt new file mode 100644 index 00000000000..c09c1469eb4 --- /dev/null +++ b/Core/Libraries/Source/DataChunk/CMakeLists.txt @@ -0,0 +1,27 @@ +set(DATACHUNK_SRC + "TableOfContents.cpp" + "DataChunkOutput.cpp" + "DataChunkInput.cpp" +) + +set(DATACHUNK_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../Include/DataChunk") + +add_library(core_datachunk STATIC ${DATACHUNK_SRC}) + +target_include_directories(core_datachunk PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}/../../Include" +) + +target_link_libraries(core_datachunk PUBLIC + core_utility +) + +# VC6 doesn't support target_compile_features +if(NOT IS_VS6_BUILD) + target_compile_features(core_datachunk PUBLIC cxx_std_98) +endif() + +if(WIN32 OR "${CMAKE_SYSTEM}" MATCHES "Windows") + target_compile_definitions(core_datachunk PRIVATE _CRT_SECURE_NO_WARNINGS) +endif() + diff --git a/Core/Libraries/Source/DataChunk/DataChunkInput.cpp b/Core/Libraries/Source/DataChunk/DataChunkInput.cpp new file mode 100644 index 00000000000..293764a76ac --- /dev/null +++ b/Core/Libraries/Source/DataChunk/DataChunkInput.cpp @@ -0,0 +1,383 @@ +/* +** Command & Conquer Generals(tm) +** Copyright 2025 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +// DataChunkInput.cpp +// Implementation of DataChunkInput +// TheSuperHackers @feature bobtista 14/11/2025 Extract chunk I/O to platform-neutral library + +#include "DataChunk/DataChunk.h" +#include +#include + +namespace DataChunk { + +//---------------------------------------------------------------------- +// DataChunkInput +//---------------------------------------------------------------------- + +DataChunkInput::DataChunkInput(DataChunkInputStream* pStream) : + m_file(pStream), + m_userData(NULL), + m_currentObject(NULL), + m_chunkStack(NULL), + m_parserList(NULL) +{ + m_contents.read(*m_file); + m_fileposOfFirstChunk = (ChunkInt)(int)m_file->tell(); +} + +DataChunkInput::~DataChunkInput() +{ + clearChunkStack(); + + UserParser* p, *next; + for (p = m_parserList; p; p = next) + { + next = p->next; + delete p; + } +} + +void DataChunkInput::clearChunkStack() +{ + InputChunk* c, *next; + + for (c = m_chunkStack; c; c = next) + { + next = c->next; + delete c; + } + + m_chunkStack = NULL; +} + +void DataChunkInput::registerParser(const ChunkString& label, const ChunkString& parentLabel, + DataChunkParserPtr parser, void* userData) +{ + UserParser* p = new UserParser(); + + p->label = label; + p->parentLabel = parentLabel; + p->parser = parser; + p->userData = userData; + p->next = m_parserList; + m_parserList = p; +} + +bool DataChunkInput::parse(void* userData) +{ + ChunkString label; + ChunkString parentLabel; + DataChunkVersionType ver; + UserParser* parser; + bool scopeOK; + DataChunkInfo info; + + if (!m_contents.isOpenedForRead()) + { + return false; + } + + if (m_chunkStack) + parentLabel = m_contents.getName(m_chunkStack->id); + + while (!atEndOfFile()) + { + if (m_chunkStack) + { + if (m_chunkStack->dataLeft < CHUNK_HEADER_BYTES) + { + if (m_chunkStack->dataLeft != 0) + { + return false; + } + break; + } + } + + label = openDataChunk(&ver); + if (atEndOfFile()) + { + break; + } + + for (parser = m_parserList; parser; parser = parser->next) + { + if (parser->label == label) + { + scopeOK = true; + + if (parentLabel != parser->parentLabel) + scopeOK = false; + + if (scopeOK) + { + info.label = label; + info.parentLabel = parentLabel; + info.version = ver; + info.dataSize = getChunkDataSize(); + + if (!parser->parser(*this, &info, userData ? userData : parser->userData)) + return false; + break; + } + } + } + + closeDataChunk(); + } + + return true; +} + +bool DataChunkInput::isValidFileType() +{ + return m_contents.isOpenedForRead(); +} + +ChunkString DataChunkInput::openDataChunk(DataChunkVersionType* ver) +{ + InputChunk* c = new InputChunk(); + c->id = 0; + c->version = 0; + c->dataSize = 0; + + m_file->read((char*)&c->id, sizeof(ChunkUInt)); + decrementDataLeft(sizeof(ChunkUInt)); + + m_file->read((char*)&c->version, sizeof(DataChunkVersionType)); + decrementDataLeft(sizeof(DataChunkVersionType)); + + m_file->read((char*)&c->dataSize, sizeof(ChunkInt)); + decrementDataLeft(sizeof(ChunkInt)); + + c->dataLeft = c->dataSize; + c->chunkStart = (ChunkInt)(int)m_file->tell(); + + *ver = c->version; + + c->next = m_chunkStack; + m_chunkStack = c; + + if (atEndOfFile()) + { + return ChunkString(); + } + return m_contents.getName(c->id); +} + +void DataChunkInput::closeDataChunk() +{ + if (m_chunkStack == NULL) + { + return; + } + + if (m_chunkStack->dataLeft > 0) + { + unsigned int newPos = (unsigned int)(m_file->tell() + m_chunkStack->dataLeft); + m_file->seek(newPos); + decrementDataLeft(m_chunkStack->dataLeft); + } + + InputChunk* c = m_chunkStack; + m_chunkStack = m_chunkStack->next; + delete c; +} + +bool DataChunkInput::atEndOfFile() +{ + return m_file->eof(); +} + +bool DataChunkInput::atEndOfChunk() +{ + if (m_chunkStack) + { + if (m_chunkStack->dataLeft <= 0) + return true; + return false; + } + + return true; +} + +void DataChunkInput::reset() +{ + clearChunkStack(); + m_file->seek(m_fileposOfFirstChunk); +} + +ChunkString DataChunkInput::getChunkLabel() +{ + if (m_chunkStack == NULL) + { + return ChunkString(); + } + + return m_contents.getName(m_chunkStack->id); +} + +DataChunkVersionType DataChunkInput::getChunkVersion() +{ + if (m_chunkStack == NULL) + { + return 0; + } + + return m_chunkStack->version; +} + +ChunkUInt DataChunkInput::getChunkDataSize() +{ + if (m_chunkStack == NULL) + { + return 0; + } + + return (ChunkUInt)m_chunkStack->dataSize; +} + +ChunkUInt DataChunkInput::getChunkDataSizeLeft() +{ + if (m_chunkStack == NULL) + { + return 0; + } + + return (ChunkUInt)m_chunkStack->dataLeft; +} + +void DataChunkInput::decrementDataLeft(ChunkInt size) +{ + InputChunk* c; + + c = m_chunkStack; + while (c) + { + c->dataLeft -= size; + c = c->next; + } +} + +ChunkReal DataChunkInput::readReal() +{ + ChunkReal r; + if (m_chunkStack->dataLeft < (ChunkInt)sizeof(ChunkReal)) + { + return 0.0f; + } + m_file->read((char*)&r, sizeof(ChunkReal)); + decrementDataLeft(sizeof(ChunkReal)); + return r; +} + +ChunkInt DataChunkInput::readInt() +{ + ChunkInt i; + if (m_chunkStack->dataLeft < (ChunkInt)sizeof(ChunkInt)) + { + return 0; + } + m_file->read((char*)&i, sizeof(ChunkInt)); + decrementDataLeft(sizeof(ChunkInt)); + return i; +} + +ChunkByte DataChunkInput::readByte() +{ + ChunkByte b; + if (m_chunkStack->dataLeft < (ChunkInt)sizeof(ChunkByte)) + { + return 0; + } + m_file->read((char*)&b, sizeof(ChunkByte)); + decrementDataLeft(sizeof(ChunkByte)); + return b; +} + +void DataChunkInput::readArrayOfBytes(char* ptr, ChunkInt len) +{ + if (m_chunkStack->dataLeft < len) + { + return; + } + m_file->read(ptr, len); + decrementDataLeft(len); +} + +ChunkString DataChunkInput::readAsciiString() +{ + ChunkUShort len; + if (m_chunkStack->dataLeft < (ChunkInt)sizeof(ChunkUShort)) + { + return ChunkString(); + } + m_file->read((char*)&len, sizeof(ChunkUShort)); + decrementDataLeft(sizeof(ChunkUShort)); + + if (m_chunkStack->dataLeft < len) + { + return ChunkString(); + } + + ChunkString theString; + if (len > 0) + { + char* buffer = new char[len + 1]; + m_file->read(buffer, len); + decrementDataLeft(len); + buffer[len] = '\0'; + theString = buffer; + delete[] buffer; + } + + return theString; +} + +ChunkWideString DataChunkInput::readUnicodeString() +{ + ChunkUShort len; + if (m_chunkStack->dataLeft < (ChunkInt)sizeof(ChunkUShort)) + { + return ChunkWideString(); + } + m_file->read((char*)&len, sizeof(ChunkUShort)); + decrementDataLeft(sizeof(ChunkUShort)); + + if (m_chunkStack->dataLeft < len * sizeof(wchar_t)) + { + return ChunkWideString(); + } + + ChunkWideString theString; + if (len > 0) + { + wchar_t* buffer = new wchar_t[len + 1]; + m_file->read((char*)buffer, len * sizeof(wchar_t)); + decrementDataLeft(len * sizeof(wchar_t)); + buffer[len] = L'\0'; + theString = buffer; + delete[] buffer; + } + + return theString; +} + +} // namespace DataChunk + diff --git a/Core/Libraries/Source/DataChunk/DataChunkOutput.cpp b/Core/Libraries/Source/DataChunk/DataChunkOutput.cpp new file mode 100644 index 00000000000..a8ba891322b --- /dev/null +++ b/Core/Libraries/Source/DataChunk/DataChunkOutput.cpp @@ -0,0 +1,202 @@ +/* +** Command & Conquer Generals(tm) +** Copyright 2025 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +// DataChunkOutput.cpp +// Implementation of DataChunkOutput +// TheSuperHackers @feature bobtista 14/11/2025 Extract chunk I/O to platform-neutral library + +#include "DataChunk/DataChunk.h" +#include +#include + +namespace DataChunk { + +//---------------------------------------------------------------------- +// DataChunkOutput +//---------------------------------------------------------------------- + +static const unsigned int INITIAL_BUFFER_SIZE = 4096; + +void DataChunkOutput::growBuffer(unsigned int needed) +{ + unsigned int newSize = m_bufferSize; + if (newSize == 0) + newSize = INITIAL_BUFFER_SIZE; + + while (m_bufferPos + needed > newSize) + { + newSize *= 2; + } + + char* newBuffer = new char[newSize]; + if (m_buffer) + { + memcpy(newBuffer, m_buffer, m_bufferPos); + delete[] m_buffer; + } + m_buffer = newBuffer; + m_bufferSize = newSize; +} + +DataChunkOutput::DataChunkOutput(DataChunkOutputStream* pOut) : + m_pOut(pOut), + m_chunkStack(NULL), + m_buffer(NULL), + m_bufferSize(0), + m_bufferPos(0) +{ +} + +DataChunkOutput::~DataChunkOutput() +{ + m_contents.write(*m_pOut); + + if (m_buffer && m_bufferPos > 0) + { + m_pOut->write(m_buffer, m_bufferPos); + } + + delete[] m_buffer; +} + +void DataChunkOutput::openDataChunk(const char* name, DataChunkVersionType ver) +{ + ChunkUInt id = m_contents.allocateID(ChunkString(name)); + + OutputChunk* c = new OutputChunk(); + c->next = m_chunkStack; + m_chunkStack = c; + m_chunkStack->id = id; + + if (m_bufferPos + 10 > m_bufferSize) + { + growBuffer(10); + } + + memcpy(&m_buffer[m_bufferPos], &id, sizeof(ChunkUInt)); + m_bufferPos += sizeof(ChunkUInt); + + memcpy(&m_buffer[m_bufferPos], &ver, sizeof(DataChunkVersionType)); + m_bufferPos += sizeof(DataChunkVersionType); + + c->filepos = (ChunkInt)m_bufferPos; + + ChunkInt dummy = 0xffff; + memcpy(&m_buffer[m_bufferPos], &dummy, sizeof(ChunkInt)); + m_bufferPos += sizeof(ChunkInt); +} + +void DataChunkOutput::closeDataChunk() +{ + if (m_chunkStack == NULL) + { + return; + } + + ChunkInt here = (ChunkInt)m_bufferPos; + ChunkInt size = here - m_chunkStack->filepos - sizeof(ChunkInt); + + memcpy(&m_buffer[m_chunkStack->filepos], &size, sizeof(ChunkInt)); + + OutputChunk* c = m_chunkStack; + m_chunkStack = m_chunkStack->next; + delete c; +} + +void DataChunkOutput::writeReal(ChunkReal r) +{ + if (m_bufferPos + sizeof(ChunkReal) > m_bufferSize) + { + growBuffer(sizeof(ChunkReal)); + } + memcpy(&m_buffer[m_bufferPos], &r, sizeof(ChunkReal)); + m_bufferPos += sizeof(ChunkReal); +} + +void DataChunkOutput::writeInt(ChunkInt i) +{ + if (m_bufferPos + sizeof(ChunkInt) > m_bufferSize) + { + growBuffer(sizeof(ChunkInt)); + } + memcpy(&m_buffer[m_bufferPos], &i, sizeof(ChunkInt)); + m_bufferPos += sizeof(ChunkInt); +} + +void DataChunkOutput::writeByte(ChunkByte b) +{ + if (m_bufferPos + sizeof(ChunkByte) > m_bufferSize) + { + growBuffer(sizeof(ChunkByte)); + } + m_buffer[m_bufferPos] = b; + m_bufferPos += sizeof(ChunkByte); +} + +void DataChunkOutput::writeAsciiString(const ChunkString& theString) +{ + ChunkUShort len = (ChunkUShort)theString.length(); + if (m_bufferPos + sizeof(ChunkUShort) + len > m_bufferSize) + { + growBuffer(sizeof(ChunkUShort) + len); + } + memcpy(&m_buffer[m_bufferPos], &len, sizeof(ChunkUShort)); + m_bufferPos += sizeof(ChunkUShort); + if (len > 0) + { + memcpy(&m_buffer[m_bufferPos], theString.c_str(), len); + m_bufferPos += len; + } +} + +void DataChunkOutput::writeUnicodeString(const ChunkWideString& theString) +{ + ChunkUShort len = (ChunkUShort)theString.length(); + if (m_bufferPos + sizeof(ChunkUShort) + len * sizeof(wchar_t) > m_bufferSize) + { + growBuffer(sizeof(ChunkUShort) + len * sizeof(wchar_t)); + } + memcpy(&m_buffer[m_bufferPos], &len, sizeof(ChunkUShort)); + m_bufferPos += sizeof(ChunkUShort); + if (len > 0) + { + memcpy(&m_buffer[m_bufferPos], theString.c_str(), len * sizeof(wchar_t)); + m_bufferPos += len * sizeof(wchar_t); + } +} + +void DataChunkOutput::writeArrayOfBytes(const char* ptr, ChunkInt len) +{ + if (len > 0) + { + if (m_bufferPos + len > m_bufferSize) + { + growBuffer(len); + } + memcpy(&m_buffer[m_bufferPos], ptr, len); + m_bufferPos += len; + } +} + +ChunkUInt DataChunkOutput::allocateID(const ChunkString& name) +{ + return m_contents.allocateID(name); +} + +} // namespace DataChunk + diff --git a/Core/Libraries/Source/DataChunk/TableOfContents.cpp b/Core/Libraries/Source/DataChunk/TableOfContents.cpp new file mode 100644 index 00000000000..5a0dcaf3b6a --- /dev/null +++ b/Core/Libraries/Source/DataChunk/TableOfContents.cpp @@ -0,0 +1,179 @@ +/* +** Command & Conquer Generals(tm) +** Copyright 2025 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +// TableOfContents.cpp +// Implementation of DataChunkTableOfContents +// TheSuperHackers @feature bobtista 14/11/2025 Extract chunk I/O to platform-neutral library + +#include "DataChunk/TableOfContents.h" +#include +#include + +namespace DataChunk { + +//---------------------------------------------------------------------- +// DataChunkTableOfContents +//---------------------------------------------------------------------- + +DataChunkTableOfContents::DataChunkTableOfContents() : + m_list(NULL), + m_nextID(1), + m_listLength(0), + m_headerOpened(false) +{ +} + +DataChunkTableOfContents::~DataChunkTableOfContents() +{ + Mapping *m, *next; + + for (m = m_list; m; m = next) + { + next = m->next; + delete m; + } +} + +Mapping* DataChunkTableOfContents::findMapping(const ChunkString& name) +{ + Mapping *m; + + for (m = m_list; m; m = m->next) + { + if (name == m->name) + return m; + } + + return NULL; +} + +ChunkUInt DataChunkTableOfContents::getID(const ChunkString& name) +{ + Mapping *m = findMapping(name); + + if (m) + return m->id; + + return 0; +} + +ChunkString DataChunkTableOfContents::getName(ChunkUInt id) +{ + Mapping *m; + + for (m = m_list; m; m = m->next) + { + if (m->id == id) + return m->name; + } + + return ChunkString(); +} + +ChunkUInt DataChunkTableOfContents::allocateID(const ChunkString& name) +{ + Mapping *m = findMapping(name); + + if (m) + return m->id; + else + { + m = new Mapping(); + + m->id = m_nextID++; + m->name = name; + m->next = m_list; + m_list = m; + + m_listLength++; + + return m->id; + } +} + +void DataChunkTableOfContents::write(DataChunkOutputStream& s) +{ + Mapping *m; + unsigned char len; + + unsigned char tag[4] = {'C', 'k', 'M', 'p'}; + s.write(tag, sizeof(tag)); + + s.write((void*)&m_listLength, sizeof(ChunkInt)); + + for (m = m_list; m; m = m->next) + { + len = (unsigned char)m->name.length(); + s.write((char*)&len, sizeof(unsigned char)); + if (len > 0) + { + s.write((char*)m->name.c_str(), len); + } + s.write((char*)&m->id, sizeof(ChunkUInt)); + } +} + +void DataChunkTableOfContents::read(DataChunkInputStream& s) +{ + ChunkInt count, i; + ChunkUInt maxID = 0; + unsigned char len; + Mapping *m; + + unsigned char tag[4] = {'x', 'x', 'x', 'x'}; + s.read(tag, sizeof(tag)); + if (tag[0] != 'C' || tag[1] != 'k' || tag[2] != 'M' || tag[3] != 'p') + { + return; + } + + s.read((char*)&count, sizeof(ChunkInt)); + + for (i = 0; i < count; i++) + { + m = new Mapping(); + + s.read((char*)&len, sizeof(unsigned char)); + + if (len > 0) + { + char* buffer = new char[len + 1]; + s.read(buffer, len); + buffer[len] = '\0'; + m->name = buffer; + delete[] buffer; + } + + s.read((char*)&m->id, sizeof(ChunkUInt)); + + m->next = m_list; + m_list = m; + + m_listLength++; + + if (m->id > maxID) + maxID = m->id; + } + m_headerOpened = (count > 0 && !s.eof()); + + if (m_nextID < maxID + 1) + m_nextID = maxID + 1; +} + +} // namespace DataChunk + diff --git a/Core/Tools/CMakeLists.txt b/Core/Tools/CMakeLists.txt index f2e820ad74b..1258bf7f001 100644 --- a/Core/Tools/CMakeLists.txt +++ b/Core/Tools/CMakeLists.txt @@ -14,6 +14,10 @@ if(RTS_BUILD_CORE_EXTRAS) add_subdirectory(CRCDiff) add_subdirectory(mangler) add_subdirectory(matchbot) + # ScriptCompiler requires C++11 (nlohmann-json), skip for VC6 + if(NOT IS_VS6_BUILD) + add_subdirectory(ScriptCompiler) + endif() add_subdirectory(textureCompress) add_subdirectory(timingTest) add_subdirectory(versionUpdate) diff --git a/Core/Tools/ScriptCompiler/CMakeLists.txt b/Core/Tools/ScriptCompiler/CMakeLists.txt new file mode 100644 index 00000000000..01ac4c7ae99 --- /dev/null +++ b/Core/Tools/ScriptCompiler/CMakeLists.txt @@ -0,0 +1,34 @@ +set(SCRIPTCOMPILER_SRC + "ScriptCompiler.cpp" +) + +add_executable(core_scriptcompiler) +set_target_properties(core_scriptcompiler PROPERTIES OUTPUT_NAME scriptcompiler) + +target_sources(core_scriptcompiler PRIVATE ${SCRIPTCOMPILER_SRC}) + +# Try to find nlohmann-json from vcpkg first +find_package(nlohmann_json CONFIG) +if(nlohmann_json_FOUND) + # Use vcpkg version + target_link_libraries(core_scriptcompiler PRIVATE nlohmann_json::nlohmann_json) +else() + # Fallback: fetch from GitHub (header-only library) + include(FetchContent) + FetchContent_Declare(json + GIT_REPOSITORY https://github.com/nlohmann/json.git + GIT_TAG v3.11.3 + GIT_SHALLOW TRUE + ) + FetchContent_MakeAvailable(json) + target_link_libraries(core_scriptcompiler PRIVATE nlohmann_json::nlohmann_json) +endif() + +target_link_libraries(core_scriptcompiler PRIVATE + core_datachunk +) + +if(WIN32 OR "${CMAKE_SYSTEM}" MATCHES "Windows") + target_link_options(core_scriptcompiler PRIVATE /subsystem:console) +endif() + diff --git a/Core/Tools/ScriptCompiler/ScriptCompiler.cpp b/Core/Tools/ScriptCompiler/ScriptCompiler.cpp new file mode 100644 index 00000000000..0550e3441de --- /dev/null +++ b/Core/Tools/ScriptCompiler/ScriptCompiler.cpp @@ -0,0 +1,1060 @@ +/* +** Command & Conquer Generals Zero Hour(tm) +** Copyright 2025 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +// TheSuperHackers @feature bobtista 14/11/2025 Bidirectional SCB to JSON compiler for AI scripts. + +#include +#include +#include +#include +#include +#include +#include +#include "DataChunk/DataChunk.h" +#include "DataChunk/StreamAdapters.h" + +using json = nlohmann::json; +using namespace DataChunk; + +static void DebugLog(const char* format, ...) +{ + char buffer[1024]; + buffer[0] = 0; + va_list args; + va_start(args, format); + vsnprintf(buffer, 1024, format, args); + va_end(args); + printf("%s\n", buffer); +} +#define DEBUG_LOG(x) DebugLog x + +struct ChunkHeader +{ + uint32_t chunkID; + uint16_t version; + uint32_t dataSize; +}; + +class DataBuffer +{ +public: + DataBuffer(const uint8_t* data, size_t size) : m_data(data), m_size(size), m_pos(0) {} + DataBuffer(const std::vector& vec) : m_data(vec.data()), m_size(vec.size()), m_pos(0) {} + + bool readString(std::string& str) + { + if (m_pos + 2 > m_size) return false; + uint16_t len = m_data[m_pos] | (m_data[m_pos+1] << 8); + m_pos += 2; + if (m_pos + len > m_size) return false; + str = std::string((char*)&m_data[m_pos], len); + m_pos += len; + return true; + } + + bool readByte(uint8_t& val) + { + if (m_pos >= m_size) return false; + val = m_data[m_pos++]; + return true; + } + + bool readInt(int32_t& val) + { + if (m_pos + 4 > m_size) return false; + val = m_data[m_pos] | (m_data[m_pos+1] << 8) | + (m_data[m_pos+2] << 16) | (m_data[m_pos+3] << 24); + m_pos += 4; + return true; + } + + bool readUInt(uint32_t& val) + { + if (m_pos + 4 > m_size) return false; + val = m_data[m_pos] | (m_data[m_pos+1] << 8) | + (m_data[m_pos+2] << 16) | (m_data[m_pos+3] << 24); + m_pos += 4; + return true; + } + + bool readShort(uint16_t& val) + { + if (m_pos + 2 > m_size) return false; + val = m_data[m_pos] | (m_data[m_pos+1] << 8); + m_pos += 2; + return true; + } + + bool readChunkHeader(ChunkHeader& header) + { + if (m_pos + 10 > m_size) return false; + if (!readUInt(header.chunkID)) return false; + if (!readShort(header.version)) return false; + if (!readUInt(header.dataSize)) return false; + return true; + } + + size_t getPosition() const { return m_pos; } + size_t remaining() const { return m_size - m_pos; } + bool atEnd() const { return m_pos >= m_size; } + +private: + const uint8_t* m_data; + size_t m_size; + size_t m_pos; +}; + +void dumpHelp(const char *exe) +{ + DEBUG_LOG(("Usage:")); + DEBUG_LOG((" To convert SCB to JSON: %s -in script.scb -out script.json", exe)); + DEBUG_LOG((" To convert JSON to SCB: %s -in script.json -out script.scb", exe)); + DEBUG_LOG(("")); + DEBUG_LOG(("The tool auto-detects the format based on file extension.")); +} + +std::string chunkIDToString(uint32_t id) +{ + char buf[5]; + buf[0] = (id >> 0) & 0xFF; + buf[1] = (id >> 8) & 0xFF; + buf[2] = (id >> 16) & 0xFF; + buf[3] = (id >> 24) & 0xFF; + buf[4] = 0; + + for (int i = 0; i < 4; i++) + { + if (!isprint(buf[i])) + return std::to_string(id); + } + + return std::string(buf); +} + +uint32_t stringToChunkID(const std::string& str) +{ + if (str.length() != 4) + return 0; + + uint32_t id = 0; + id |= (uint8_t)str[0] << 0; + id |= (uint8_t)str[1] << 8; + id |= (uint8_t)str[2] << 16; + id |= (uint8_t)str[3] << 24; + return id; +} + +json parseChunkRecursive(DataBuffer& buffer, const std::vector& stringTable, int depth = 0); + +json parseScriptChunk(DataBuffer& buffer, const std::vector& stringTable) +{ + json parsed; + + std::string scriptName, comment, conditionComment, actionComment; + buffer.readString(scriptName); + buffer.readString(comment); + buffer.readString(conditionComment); + buffer.readString(actionComment); + + uint8_t isActive, isOneShot, easy, normal, hard, isSubroutine; + buffer.readByte(isActive); + buffer.readByte(isOneShot); + buffer.readByte(easy); + buffer.readByte(normal); + buffer.readByte(hard); + buffer.readByte(isSubroutine); + + int32_t delaySeconds; + buffer.readInt(delaySeconds); + + parsed["scriptName"] = scriptName; + parsed["comment"] = comment; + parsed["conditionComment"] = conditionComment; + parsed["actionComment"] = actionComment; + parsed["isActive"] = isActive != 0; + parsed["isOneShot"] = isOneShot != 0; + parsed["isSubroutine"] = isSubroutine != 0; + parsed["delayEvaluationSeconds"] = delaySeconds; + + json difficulty = json::array(); + if (easy) difficulty.push_back("easy"); + if (normal) difficulty.push_back("normal"); + if (hard) difficulty.push_back("hard"); + parsed["difficulty"] = difficulty; + + json children = json::array(); + while (!buffer.atEnd()) + { + json child = parseChunkRecursive(buffer, stringTable, 1); + if (child.is_null()) break; + children.push_back(child); + } + if (!children.empty()) + parsed["children"] = children; + + return parsed; +} + +json parseParameter(DataBuffer& buffer, const std::vector& stringTable) +{ + json param; + + int32_t paramType; + if (!buffer.readInt(paramType)) return json(); + + param["paramType"] = paramType; + + if (paramType == 32) + { + float x, y, z; + uint32_t xi, yi, zi; + buffer.readUInt(xi); + buffer.readUInt(yi); + buffer.readUInt(zi); + memcpy(&x, &xi, 4); + memcpy(&y, &yi, 4); + memcpy(&z, &zi, 4); + param["coord"] = {x, y, z}; + } + else + { + int32_t intVal; + uint32_t realBits; + std::string strVal; + + buffer.readInt(intVal); + buffer.readUInt(realBits); + + float realVal; + memcpy(&realVal, &realBits, 4); + + buffer.readString(strVal); + + param["int"] = intVal; + param["real"] = realVal; + param["string"] = strVal; + } + + return param; +} + +json parseConditionChunk(DataBuffer& buffer, const std::vector& stringTable, uint16_t version) +{ + json parsed; + + int32_t conditionType; + buffer.readInt(conditionType); + + parsed["conditionType"] = conditionType; + + if (conditionType < (int32_t)stringTable.size() && !stringTable[conditionType].empty()) + parsed["conditionTypeName"] = stringTable[conditionType]; + else + parsed["conditionTypeName"] = "UNKNOWN_" + std::to_string(conditionType); + + if (version >= 4) + { + uint32_t nameKey; + buffer.readUInt(nameKey); + parsed["nameKey"] = nameKey; + } + + int32_t numParams; + buffer.readInt(numParams); + parsed["numParameters"] = numParams; + + json params = json::array(); + for (int i = 0; i < numParams && !buffer.atEnd(); i++) + { + json param = parseParameter(buffer, stringTable); + if (!param.is_null()) + params.push_back(param); + } + parsed["parameters"] = params; + + return parsed; +} + +json parseActionChunk(DataBuffer& buffer, const std::vector& stringTable, uint16_t version) +{ + json parsed; + + int32_t actionType; + buffer.readInt(actionType); + + parsed["actionType"] = actionType; + + if (actionType < (int32_t)stringTable.size() && !stringTable[actionType].empty()) + parsed["actionTypeName"] = stringTable[actionType]; + else + parsed["actionTypeName"] = "UNKNOWN_" + std::to_string(actionType); + + if (version >= 2) + { + uint32_t nameKey; + buffer.readUInt(nameKey); + parsed["nameKey"] = nameKey; + } + + int32_t numParams; + buffer.readInt(numParams); + parsed["numParameters"] = numParams; + + json params = json::array(); + for (int i = 0; i < numParams && !buffer.atEnd(); i++) + { + json param = parseParameter(buffer, stringTable); + if (!param.is_null()) + params.push_back(param); + } + parsed["parameters"] = params; + + return parsed; +} + +json parseChunkRecursive(DataBuffer& buffer, const std::vector& stringTable, int depth) +{ + if (buffer.atEnd() || depth > 20) + return json(); + + ChunkHeader header; + size_t headerStart = buffer.getPosition(); + if (!buffer.readChunkHeader(header)) + return json(); + + std::string chunkTypeName = "UNKNOWN"; + if (header.chunkID < stringTable.size() && !stringTable[header.chunkID].empty()) + chunkTypeName = stringTable[header.chunkID]; + + size_t dataStart = buffer.getPosition(); + size_t dataEnd = dataStart + header.dataSize; + + if (dataEnd > buffer.getPosition() + buffer.remaining()) + return json(); + + std::vector chunkData(header.dataSize); + for (size_t i = 0; i < header.dataSize && !buffer.atEnd(); i++) + buffer.readByte(chunkData[i]); + + DataBuffer chunkBuffer(chunkData); + json parsed; + + DEBUG_LOG(("%*sChunk: %s (ID=%u) v%d size=%u", depth*2, "", + chunkTypeName.c_str(), header.chunkID, header.version, header.dataSize)); + + if (chunkTypeName == "Script") + { + parsed = parseScriptChunk(chunkBuffer, stringTable); + parsed["type"] = "Script"; + } + else if (chunkTypeName == "ScriptAction" || chunkTypeName == "ScriptActionFalse") + { + parsed = parseActionChunk(chunkBuffer, stringTable, header.version); + parsed["type"] = chunkTypeName; + } + else if (chunkTypeName == "Condition") + { + parsed = parseConditionChunk(chunkBuffer, stringTable, header.version); + parsed["type"] = "Condition"; + } + else if (chunkTypeName == "ScriptGroup") + { + std::string groupName; + uint8_t isActive, isSubroutine = 0; + + chunkBuffer.readString(groupName); + chunkBuffer.readByte(isActive); + if (header.version >= 2) + chunkBuffer.readByte(isSubroutine); + + parsed["type"] = "ScriptGroup"; + parsed["groupName"] = groupName; + parsed["isActive"] = isActive != 0; + parsed["isSubroutine"] = isSubroutine != 0; + + json children = json::array(); + while (!chunkBuffer.atEnd()) + { + json child = parseChunkRecursive(chunkBuffer, stringTable, depth + 1); + if (child.is_null()) break; + children.push_back(child); + } + if (!children.empty()) + parsed["children"] = children; + } + else if (chunkTypeName == "PlayerScriptsList" || chunkTypeName == "ScriptList" || + chunkTypeName == "OrCondition") + { + parsed["type"] = chunkTypeName; + json children = json::array(); + while (!chunkBuffer.atEnd()) + { + json child = parseChunkRecursive(chunkBuffer, stringTable, depth + 1); + if (child.is_null()) break; + children.push_back(child); + } + if (!children.empty()) + parsed["children"] = children; + } + else + { + parsed["type"] = chunkTypeName; + parsed["rawData"] = chunkData; + } + + parsed["_meta"] = { + {"chunkID", header.chunkID}, + {"version", header.version}, + {"dataSize", header.dataSize} + }; + + return parsed; +} + +struct ParseContext +{ + json* output; + std::vector* stringTable; +}; + +bool parseChunkCallback(DataChunkInput& file, DataChunkInfo* info, void* userData) +{ + ParseContext* ctx = (ParseContext*)userData; + json chunk; + + std::string chunkTypeName = info->label; + + uint32_t chunkID = 0; + for (size_t i = 0; i < ctx->stringTable->size(); i++) + { + if ((*ctx->stringTable)[i] == chunkTypeName) + { + chunkID = (uint32_t)i; + break; + } + } + + chunk["id"] = chunkID; + chunk["type"] = chunkTypeName; + chunk["version"] = info->version; + chunk["dataSize"] = info->dataSize; + + if (chunkTypeName == "Script" || chunkTypeName == "ScriptAction" || + chunkTypeName == "ScriptActionFalse" || chunkTypeName == "Condition" || + chunkTypeName == "ScriptGroup" || chunkTypeName == "PlayerScriptsList" || + chunkTypeName == "ScriptList" || chunkTypeName == "OrCondition") + { + std::vector rawData(info->dataSize); + file.readArrayOfBytes((char*)rawData.data(), info->dataSize); + + DataBuffer chunkBuffer(rawData); + json parsed = parseChunkRecursive(chunkBuffer, *ctx->stringTable, 0); + if (!parsed.is_null()) + chunk["parsed"] = parsed; + else + chunk["rawData"] = rawData; + } + else + { + std::vector rawData(info->dataSize); + file.readArrayOfBytes((char*)rawData.data(), info->dataSize); + chunk["rawData"] = rawData; + } + + ctx->output->push_back(chunk); + return true; +} + +bool readBinaryToJson(const std::string& inFile, json& output) +{ + DEBUG_LOG(("Reading binary SCB file: %s", inFile.c_str())); + + FILE* fp = fopen(inFile.c_str(), "rb"); + if (!fp) + { + DEBUG_LOG(("Failed to open file: %s", inFile.c_str())); + return false; + } + + char magic[4]; + if (fread(magic, 1, 4, fp) != 4) + { + fclose(fp); + return false; + } + + std::vector stringTable; + if (memcmp(magic, "CkMp", 4) == 0) + { + uint32_t numStrings; + if (fread(&numStrings, 4, 1, fp) != 1) + { + fclose(fp); + return false; + } + + DEBUG_LOG(("File header: CkMp, %u strings in table", numStrings)); + + for (uint32_t i = 0; i < numStrings; i++) + { + uint8_t length; + if (fread(&length, 1, 1, fp) != 1) + { + fclose(fp); + return false; + } + + std::vector buffer(length); + if (length > 0) + { + if (fread(buffer.data(), 1, length, fp) != length) + { + fclose(fp); + return false; + } + } + + uint32_t stringID; + if (fread(&stringID, 4, 1, fp) != 1) + { + fclose(fp); + return false; + } + + std::string str(buffer.begin(), buffer.end()); + + if (stringID >= stringTable.size()) + stringTable.resize(stringID + 1); + + stringTable[stringID] = str; + DEBUG_LOG((" String ID %u: '%s'", stringID, str.c_str())); + } + } + + fseek(fp, 0, SEEK_SET); + + FileInputStream stream(fp); + DataChunkInput chunkInput(&stream); + + if (!chunkInput.isValidFileType()) + { + DEBUG_LOG(("Failed to read file header")); + fclose(fp); + return false; + } + + DEBUG_LOG(("String table has %zu entries", stringTable.size())); + + json chunks = json::array(); + ParseContext ctx; + ctx.output = &chunks; + ctx.stringTable = &stringTable; + + chunkInput.registerParser("Script", "", parseChunkCallback, &ctx); + chunkInput.registerParser("ScriptGroup", "", parseChunkCallback, &ctx); + chunkInput.registerParser("ScriptAction", "", parseChunkCallback, &ctx); + chunkInput.registerParser("ScriptActionFalse", "", parseChunkCallback, &ctx); + chunkInput.registerParser("Condition", "", parseChunkCallback, &ctx); + chunkInput.registerParser("OrCondition", "", parseChunkCallback, &ctx); + chunkInput.registerParser("ScriptList", "", parseChunkCallback, &ctx); + chunkInput.registerParser("PlayerScriptsList", "", parseChunkCallback, &ctx); + + if (!chunkInput.parse(&ctx)) + { + DEBUG_LOG(("Failed to parse chunks")); + fclose(fp); + return false; + } + + fclose(fp); + + output["format"] = "SCB"; + output["version"] = 1; + output["stringTable"] = stringTable; + output["chunks"] = chunks; + + DEBUG_LOG(("Successfully read %zu chunks", chunks.size())); + return true; +} + +std::vector serializeParameter(const json& param) +{ + std::vector data; + + int32_t paramType = param.value("paramType", 0); + data.resize(4); + memcpy(data.data(), ¶mType, 4); + + if (paramType == 32 && param.contains("coord")) + { + auto coord = param["coord"]; + float x = coord[0].get(); + float y = coord[1].get(); + float z = coord[2].get(); + + size_t offset = data.size(); + data.resize(offset + 12); + memcpy(&data[offset], &x, 4); + memcpy(&data[offset + 4], &y, 4); + memcpy(&data[offset + 8], &z, 4); + } + else + { + int32_t intVal = param.value("int", 0); + float realVal = param.value("real", 0.0f); + std::string strVal = param.value("string", ""); + + size_t offset = data.size(); + data.resize(offset + 4); + memcpy(&data[offset], &intVal, 4); + + offset = data.size(); + data.resize(offset + 4); + memcpy(&data[offset], &realVal, 4); + + uint16_t strLen = strVal.length(); + offset = data.size(); + data.resize(offset + 2 + strLen); + memcpy(&data[offset], &strLen, 2); + if (strLen > 0) + memcpy(&data[offset + 2], strVal.c_str(), strLen); + } + + return data; +} + +std::vector serializeChunk(const json& chunk, const std::vector& stringTable); + +std::vector serializeCondition(const json& cond, const std::vector& stringTable) +{ + std::vector data; + + int32_t condType = cond.value("conditionType", 0); + data.resize(4); + memcpy(data.data(), &condType, 4); + + uint16_t version = cond["_meta"].value("version", 4); + if (version >= 4) + { + uint32_t nameKey = cond.value("nameKey", 0); + size_t offset = data.size(); + data.resize(offset + 4); + memcpy(&data[offset], &nameKey, 4); + } + + int32_t numParams = cond.value("numParameters", 0); + size_t offset = data.size(); + data.resize(offset + 4); + memcpy(&data[offset], &numParams, 4); + + if (cond.contains("parameters")) + { + for (const auto& param : cond["parameters"]) + { + auto paramData = serializeParameter(param); + data.insert(data.end(), paramData.begin(), paramData.end()); + } + } + + return data; +} + +std::vector serializeAction(const json& action, const std::vector& stringTable) +{ + std::vector data; + + int32_t actionType = action.value("actionType", 0); + data.resize(4); + memcpy(data.data(), &actionType, 4); + + uint16_t version = action["_meta"].value("version", 2); + if (version >= 2) + { + uint32_t nameKey = action.value("nameKey", 0); + size_t offset = data.size(); + data.resize(offset + 4); + memcpy(&data[offset], &nameKey, 4); + } + + int32_t numParams = action.value("numParameters", 0); + size_t offset = data.size(); + data.resize(offset + 4); + memcpy(&data[offset], &numParams, 4); + + if (action.contains("parameters")) + { + for (const auto& param : action["parameters"]) + { + auto paramData = serializeParameter(param); + data.insert(data.end(), paramData.begin(), paramData.end()); + } + } + + return data; +} + +std::vector serializeScript(const json& script, const std::vector& stringTable) +{ + std::vector data; + + std::string scriptName = script.value("scriptName", ""); + std::string comment = script.value("comment", ""); + std::string condComment = script.value("conditionComment", ""); + std::string actComment = script.value("actionComment", ""); + + auto writeStr = [&](const std::string& s) { + uint16_t len = s.length(); + size_t offset = data.size(); + data.resize(offset + 2 + len); + memcpy(&data[offset], &len, 2); + if (len > 0) + memcpy(&data[offset + 2], s.c_str(), len); + }; + + writeStr(scriptName); + writeStr(comment); + writeStr(condComment); + writeStr(actComment); + + uint8_t isActive = script.value("isActive", true) ? 1 : 0; + uint8_t isOneShot = script.value("isOneShot", false) ? 1 : 0; + uint8_t easy = script.value("easy", true) ? 1 : 0; + uint8_t normal = script.value("normal", true) ? 1 : 0; + uint8_t hard = script.value("hard", true) ? 1 : 0; + uint8_t isSubroutine = script.value("isSubroutine", false) ? 1 : 0; + + data.push_back(isActive); + data.push_back(isOneShot); + data.push_back(easy); + data.push_back(normal); + data.push_back(hard); + data.push_back(isSubroutine); + + int32_t delaySeconds = script.value("delayEvaluationSeconds", 0); + size_t offset = data.size(); + data.resize(offset + 4); + memcpy(&data[offset], &delaySeconds, 4); + + if (script.contains("children")) + { + for (const auto& child : script["children"]) + { + auto childData = serializeChunk(child, stringTable); + data.insert(data.end(), childData.begin(), childData.end()); + } + } + + return data; +} + +std::vector serializeScriptGroup(const json& group, const std::vector& stringTable) +{ + std::vector data; + + std::string groupName = group.value("groupName", ""); + uint16_t len = groupName.length(); + data.resize(2 + len); + memcpy(data.data(), &len, 2); + if (len > 0) + memcpy(&data[2], groupName.c_str(), len); + + uint8_t isActive = group.value("isActive", true) ? 1 : 0; + uint8_t isSubroutine = group.value("isSubroutine", false) ? 1 : 0; + data.push_back(isActive); + data.push_back(isSubroutine); + + if (group.contains("children")) + { + for (const auto& child : group["children"]) + { + auto childData = serializeChunk(child, stringTable); + data.insert(data.end(), childData.begin(), childData.end()); + } + } + + return data; +} + +std::vector serializeChunk(const json& chunk, const std::vector& stringTable) +{ + std::vector result; + std::vector chunkData; + + std::string chunkType = chunk.value("type", ""); + uint32_t chunkID = chunk["_meta"].value("chunkID", 0); + uint16_t version = chunk["_meta"].value("version", 1); + + if (chunkType == "Script") + { + chunkData = serializeScript(chunk, stringTable); + } + else if (chunkType == "ScriptAction" || chunkType == "ScriptActionFalse") + { + chunkData = serializeAction(chunk, stringTable); + } + else if (chunkType == "Condition") + { + chunkData = serializeCondition(chunk, stringTable); + } + else if (chunkType == "ScriptGroup") + { + chunkData = serializeScriptGroup(chunk, stringTable); + } + else if (chunk.contains("children")) + { + for (const auto& child : chunk["children"]) + { + auto childData = serializeChunk(child, stringTable); + chunkData.insert(chunkData.end(), childData.begin(), childData.end()); + } + } + else if (chunk.contains("rawData")) + { + chunkData = chunk["rawData"].get>(); + } + + uint32_t dataSize = chunkData.size(); + result.resize(10 + dataSize); + + memcpy(&result[0], &chunkID, 4); + memcpy(&result[4], &version, 2); + memcpy(&result[6], &dataSize, 4); + if (dataSize > 0) + memcpy(&result[10], chunkData.data(), dataSize); + + return result; +} + +bool writeJsonToBinary(const json& input, const std::string& outFile) +{ + DEBUG_LOG(("Writing binary SCB file: %s", outFile.c_str())); + + FILE* fp = fopen(outFile.c_str(), "wb"); + if (!fp) + { + DEBUG_LOG(("Failed to create file: %s", outFile.c_str())); + return false; + } + + { + FileOutputStream stream(fp); + DataChunkOutput chunkOutput(&stream); + + if (input.contains("stringTable")) + { + std::vector stringTable = input["stringTable"].get>(); + + for (size_t i = 0; i < stringTable.size(); i++) + { + if (!stringTable[i].empty()) + { + chunkOutput.allocateID(stringTable[i]); + } + } + + if (input.contains("chunks")) + { + for (const auto& chunkEntry : input["chunks"]) + { + std::string chunkTypeName = chunkEntry.value("type", ""); + if (chunkTypeName.empty()) + { + uint32_t chunkID = chunkEntry.value("id", 0); + if (chunkID < stringTable.size()) + chunkTypeName = stringTable[chunkID]; + } + + uint16_t outerVersion = chunkEntry.value("version", 1); + + if (chunkTypeName.empty()) + chunkTypeName = "UNKNOWN"; + + chunkOutput.openDataChunk(chunkTypeName.c_str(), outerVersion); + + if (chunkEntry.contains("parsed")) + { + const json& parsed = chunkEntry["parsed"]; + std::vector innerData = serializeChunk(parsed, stringTable); + if (!innerData.empty()) + chunkOutput.writeArrayOfBytes((const char*)innerData.data(), innerData.size()); + } + else if (chunkEntry.contains("rawData")) + { + std::vector innerData = chunkEntry["rawData"].get>(); + if (!innerData.empty()) + chunkOutput.writeArrayOfBytes((const char*)innerData.data(), innerData.size()); + } + + chunkOutput.closeDataChunk(); + } + } + } + } + + fclose(fp); + DEBUG_LOG(("Successfully wrote binary SCB file")); + return true; +} + +bool readJsonFile(const std::string& inFile, json& output) +{ + FILE* fp = fopen(inFile.c_str(), "r"); + if (!fp) + { + DEBUG_LOG(("Cannot open JSON file: %s", inFile.c_str())); + return false; + } + + fseek(fp, 0, SEEK_END); + size_t size = ftell(fp); + fseek(fp, 0, SEEK_SET); + + std::vector buffer(size + 1); + size_t numRead = fread(buffer.data(), 1, size, fp); + fclose(fp); + + if (numRead != size) + { + DEBUG_LOG(("Failed to read JSON file: %s", inFile.c_str())); + return false; + } + + buffer[size] = 0; + + try + { + output = json::parse(buffer.data()); + return true; + } + catch (const json::exception& e) + { + DEBUG_LOG(("JSON parse error: %s", e.what())); + return false; + } +} + +bool writeJsonFile(const json& data, const std::string& outFile) +{ + FILE* fp = fopen(outFile.c_str(), "w"); + if (!fp) + { + DEBUG_LOG(("Cannot create JSON file: %s", outFile.c_str())); + return false; + } + + std::string jsonStr = data.dump(2); + size_t written = fwrite(jsonStr.c_str(), 1, jsonStr.length(), fp); + fclose(fp); + + if (written != jsonStr.length()) + { + DEBUG_LOG(("Failed to write JSON file: %s", outFile.c_str())); + return false; + } + + return true; +} + +bool endsWithIgnoreCase(const std::string& str, const std::string& suffix) +{ + if (str.length() < suffix.length()) + return false; + + size_t start = str.length() - suffix.length(); + for (size_t i = 0; i < suffix.length(); ++i) + { + char c1 = tolower(str[start + i]); + char c2 = tolower(suffix[i]); + if (c1 != c2) + return false; + } + return true; +} + +int main(int argc, char **argv) +{ + std::string inFile; + std::string outFile; + + for (int i = 1; i < argc; ++i) + { + if (!strcmp(argv[i], "-help") || !strcmp(argv[i], "--help")) + { + dumpHelp(argv[0]); + return EXIT_SUCCESS; + } + + if (!strcmp(argv[i], "-in")) + { + ++i; + if (i < argc) + inFile = argv[i]; + } + + if (!strcmp(argv[i], "-out")) + { + ++i; + if (i < argc) + outFile = argv[i]; + } + } + + if (inFile.empty() || outFile.empty()) + { + dumpHelp(argv[0]); + return EXIT_FAILURE; + } + + bool inIsScb = endsWithIgnoreCase(inFile, ".scb"); + bool outIsScb = endsWithIgnoreCase(outFile, ".scb"); + bool inIsJson = endsWithIgnoreCase(inFile, ".json"); + bool outIsJson = endsWithIgnoreCase(outFile, ".json"); + + if (inIsScb && outIsJson) + { + json data; + if (!readBinaryToJson(inFile, data)) + return EXIT_FAILURE; + + if (!writeJsonFile(data, outFile)) + return EXIT_FAILURE; + + DEBUG_LOG(("Successfully converted %s to %s", inFile.c_str(), outFile.c_str())); + return EXIT_SUCCESS; + } + else if (inIsJson && outIsScb) + { + json data; + if (!readJsonFile(inFile, data)) + return EXIT_FAILURE; + + if (!writeJsonToBinary(data, outFile)) + return EXIT_FAILURE; + + DEBUG_LOG(("Successfully converted %s to %s", inFile.c_str(), outFile.c_str())); + return EXIT_SUCCESS; + } + else + { + DEBUG_LOG(("ERROR: Cannot determine conversion direction from file extensions")); + DEBUG_LOG(("Input: %s, Output: %s", inFile.c_str(), outFile.c_str())); + dumpHelp(argv[0]); + return EXIT_FAILURE; + } +} + diff --git a/vcpkg.json b/vcpkg.json index 011b913c8aa..55e227ae424 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -3,6 +3,7 @@ "builtin-baseline": "b02e341c927f16d991edbd915d8ea43eac52096c", "dependencies": [ "zlib", - "ffmpeg" + "ffmpeg", + "nlohmann-json" ] } \ No newline at end of file