diff --git a/src/BinaryDict.cpp b/src/BinaryDict.cpp index 29ca9cd88..9c4dc5581 100644 --- a/src/BinaryDict.cpp +++ b/src/BinaryDict.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include "BinaryDict.hpp" @@ -33,6 +34,58 @@ size_t BinaryDict::KeyMaxLength() const { return maxLength; } +namespace { + +// All integer fields are serialized as fixed-width uint32_t so that the file +// is identical regardless of the word size of the platform that built it. +// Files written by old 64-bit builds used native 8-byte size_t fields; the +// deserializers below detect and accept both layouts. +void WriteField(FILE* fp, size_t value) { + if (value > UINT32_MAX) { + throw InvalidFormat("OpenCC binary dictionary field exceeds uint32 range"); + } + uint32_t field = static_cast(value); + fwrite(&field, sizeof(field), 1, fp); +} + +uint64_t ReadField(const char* data, size_t size, size_t* offset, + size_t fieldWidth) { + if (fieldWidth > size || *offset > size - fieldWidth) { + throw InvalidFormat("Invalid OpenCC binary dictionary (truncated)"); + } + uint64_t value; + if (fieldWidth == sizeof(uint32_t)) { + uint32_t field; + memcpy(&field, data + *offset, sizeof(field)); + value = field; + } else { + memcpy(&value, data + *offset, sizeof(value)); + } + *offset += fieldWidth; + return value; +} + +// Detects the integer field width of a serialized BinaryDict. +// - Fixed-width layout (current builds and old 32-bit builds): the first 8 +// bytes are numItems followed by keyTotalLength; keyTotalLength is +// non-zero whenever the dictionary has entries, and an empty dictionary +// is exactly 12 bytes long. +// - Legacy 64-bit layout: the first field is an 8-byte numItems whose high +// 4 bytes are zero for any realistic dictionary. +size_t DetectFieldWidth(const char* data, size_t size) { + if (size < 8) { + return sizeof(uint32_t); + } + const bool highBytesZero = + data[4] == 0 && data[5] == 0 && data[6] == 0 && data[7] == 0; + if (highBytesZero && size != 3 * sizeof(uint32_t)) { + return sizeof(uint64_t); + } + return sizeof(uint32_t); +} + +} // namespace + void BinaryDict::SerializeToFile(FILE* fp) const { std::string keyBuf, valueBuf; std::vector keyOffsets, valueOffsets; @@ -41,26 +94,26 @@ void BinaryDict::SerializeToFile(FILE* fp) const { valueTotalLength); // Number of items size_t numItems = lexicon->Length(); - fwrite(&numItems, sizeof(size_t), 1, fp); + WriteField(fp, numItems); // Data - fwrite(&keyTotalLength, sizeof(size_t), 1, fp); + WriteField(fp, keyTotalLength); fwrite(keyBuf.c_str(), sizeof(char), keyTotalLength, fp); - fwrite(&valueTotalLength, sizeof(size_t), 1, fp); + WriteField(fp, valueTotalLength); fwrite(valueBuf.c_str(), sizeof(char), valueTotalLength, fp); size_t keyCursor = 0, valueCursor = 0; for (const std::unique_ptr& entry : *lexicon) { // Number of values size_t numValues = entry->NumValues(); - fwrite(&numValues, sizeof(size_t), 1, fp); + WriteField(fp, numValues); // Key offset size_t keyOffset = keyOffsets[keyCursor++]; - fwrite(&keyOffset, sizeof(size_t), 1, fp); + WriteField(fp, keyOffset); // Values offset for (size_t i = 0; i < numValues; i++) { size_t valueOffset = valueOffsets[valueCursor++]; - fwrite(&valueOffset, sizeof(size_t), 1, fp); + WriteField(fp, valueOffset); } } assert(keyCursor == numItems); @@ -69,112 +122,39 @@ void BinaryDict::SerializeToFile(FILE* fp) const { BinaryDictPtr BinaryDict::NewFromFile(FILE* fp) { long savedOffset = ftell(fp); fseek(fp, 0L, SEEK_END); - long offsetBoundLong = ftell(fp) - savedOffset; + long remainingSizeLong = ftell(fp) - savedOffset; fseek(fp, savedOffset, SEEK_SET); - assert(offsetBoundLong >= 0); - size_t offsetBound = static_cast(offsetBoundLong); - - BinaryDictPtr dict(new BinaryDict(LexiconPtr(new Lexicon))); - - // Number of items - size_t numItems; - size_t unitsRead = fread(&numItems, sizeof(size_t), 1, fp); - if (unitsRead != 1) { - throw InvalidFormat("Invalid OpenCC binary dictionary (numItems)"); - } - - // Keys - size_t keyTotalLength; - unitsRead = fread(&keyTotalLength, sizeof(size_t), 1, fp); - if (unitsRead != 1) { - throw InvalidFormat("Invalid OpenCC binary dictionary (keyTotalLength)"); - } - if (keyTotalLength > offsetBound) { - throw InvalidFormat("Invalid OpenCC binary dictionary (keyTotalLength exceeds file size)"); - } - dict->keyBuffer.resize(keyTotalLength); - unitsRead = fread(const_cast(dict->keyBuffer.c_str()), sizeof(char), - keyTotalLength, fp); - if (unitsRead != keyTotalLength) { - throw InvalidFormat("Invalid OpenCC binary dictionary (keyBuffer)"); - } - - // Values - size_t valueTotalLength; - unitsRead = fread(&valueTotalLength, sizeof(size_t), 1, fp); - if (unitsRead != 1) { - throw InvalidFormat("Invalid OpenCC binary dictionary (valueTotalLength)"); - } - if (valueTotalLength > offsetBound) { - throw InvalidFormat( - "Invalid OpenCC binary dictionary (valueTotalLength exceeds file size)"); - } - dict->valueBuffer.resize(valueTotalLength); - unitsRead = fread(const_cast(dict->valueBuffer.c_str()), sizeof(char), - valueTotalLength, fp); - if (unitsRead != valueTotalLength) { - throw InvalidFormat("Invalid OpenCC binary dictionary (valueBuffer)"); - } + assert(remainingSizeLong >= 0); + size_t remainingSize = static_cast(remainingSizeLong); - // Offsets - for (size_t i = 0; i < numItems; i++) { - // Number of values - size_t numValues; - unitsRead = fread(&numValues, sizeof(size_t), 1, fp); - if (unitsRead != 1) { - throw InvalidFormat("Invalid OpenCC binary dictionary (numValues)"); - } - // Key offset - size_t keyOffset; - unitsRead = fread(&keyOffset, sizeof(size_t), 1, fp); - if (unitsRead != 1 || keyOffset >= keyTotalLength) { - throw InvalidFormat("Invalid OpenCC binary dictionary (keyOffset)"); - } - const char* keyStart = dict->keyBuffer.c_str() + keyOffset; - if (memchr(keyStart, '\0', keyTotalLength - keyOffset) == nullptr) { - throw InvalidFormat( - "Invalid OpenCC binary dictionary (key not null-terminated)"); - } - std::string key = keyStart; - // Value offset - std::vector values; - for (size_t j = 0; j < numValues; j++) { - size_t valueOffset; - unitsRead = fread(&valueOffset, sizeof(size_t), 1, fp); - if (unitsRead != 1 || valueOffset >= valueTotalLength) { - throw InvalidFormat("Invalid OpenCC binary dictionary (valueOffset)"); - } - const char* valueStart = dict->valueBuffer.c_str() + valueOffset; - if (memchr(valueStart, '\0', valueTotalLength - valueOffset) == nullptr) { - throw InvalidFormat( - "Invalid OpenCC binary dictionary (value not null-terminated)"); - } - values.push_back(valueStart); - } - DictEntry* entry = DictEntryFactory::New(key, values); - dict->lexicon->Add(entry); + std::string buffer; + buffer.resize(remainingSize); + size_t bytesRead = fread(const_cast(buffer.c_str()), sizeof(char), + remainingSize, fp); + if (bytesRead != remainingSize) { + throw InvalidFormat("Invalid OpenCC binary dictionary (truncated)"); } - - return dict; + return NewFromBuffer(buffer.data(), buffer.size()); } BinaryDictPtr BinaryDict::NewFromBuffer(const char* data, size_t size) { BinaryDictPtr dict(new BinaryDict(LexiconPtr(new Lexicon))); size_t offset = 0; + const size_t fieldWidth = DetectFieldWidth(data, size); - auto readSizeT = [&]() -> size_t { - if (offset + sizeof(size_t) > size) { - throw InvalidFormat("Invalid OpenCC binary dictionary (truncated)"); + auto readField = [&]() -> size_t { + uint64_t value = ReadField(data, size, &offset, fieldWidth); + if (value > size) { + // Every field is a count, length or offset bounded by the data size; + // this also guarantees the value fits in size_t on 32-bit platforms. + throw InvalidFormat("Invalid OpenCC binary dictionary (field too large)"); } - size_t val; - memcpy(&val, data + offset, sizeof(size_t)); - offset += sizeof(size_t); - return val; + return static_cast(value); }; - size_t numItems = readSizeT(); + size_t numItems = readField(); - size_t keyTotalLength = readSizeT(); + size_t keyTotalLength = readField(); if (keyTotalLength > size - offset) { throw InvalidFormat( "Invalid OpenCC binary dictionary (keyTotalLength exceeds data size)"); @@ -182,7 +162,7 @@ BinaryDictPtr BinaryDict::NewFromBuffer(const char* data, size_t size) { dict->keyBuffer.assign(data + offset, keyTotalLength); offset += keyTotalLength; - size_t valueTotalLength = readSizeT(); + size_t valueTotalLength = readField(); if (valueTotalLength > size - offset) { throw InvalidFormat( "Invalid OpenCC binary dictionary (valueTotalLength exceeds data size)"); @@ -191,8 +171,8 @@ BinaryDictPtr BinaryDict::NewFromBuffer(const char* data, size_t size) { offset += valueTotalLength; for (size_t i = 0; i < numItems; i++) { - size_t numValues = readSizeT(); - size_t keyOffset = readSizeT(); + size_t numValues = readField(); + size_t keyOffset = readField(); if (keyOffset >= keyTotalLength) { throw InvalidFormat("Invalid OpenCC binary dictionary (keyOffset)"); } @@ -204,7 +184,7 @@ BinaryDictPtr BinaryDict::NewFromBuffer(const char* data, size_t size) { std::string key = keyStart; std::vector values; for (size_t j = 0; j < numValues; j++) { - size_t valueOffset = readSizeT(); + size_t valueOffset = readField(); if (valueOffset >= valueTotalLength) { throw InvalidFormat("Invalid OpenCC binary dictionary (valueOffset)"); } diff --git a/src/BinaryDictTest.cpp b/src/BinaryDictTest.cpp index 0c64baafa..0d155e7a3 100644 --- a/src/BinaryDictTest.cpp +++ b/src/BinaryDictTest.cpp @@ -27,23 +27,31 @@ class BinaryDictTest : public TextDictTestBase { : binDict(new BinaryDict(textDict->GetLexicon())), fileName("dict.bin"){}; // Write a crafted binary file for BinaryDict with controllable fields. - static std::string WriteMalformedBinaryDict( - size_t numItems, size_t keyTotalLength, const std::string& keyBuffer, - size_t valueTotalLength, const std::string& valueBuffer, - const std::vector>>& + // FIELD selects the integer field width: uint32_t for the fixed-width + // layout written by current builds, uint64_t for the legacy layout written + // by old 64-bit builds. + template + static std::string WriteCraftedBinaryDict( + uint64_t numItems, uint64_t keyTotalLength, const std::string& keyBuffer, + uint64_t valueTotalLength, const std::string& valueBuffer, + const std::vector>>& items) { - const std::string path = "malformed_binary_dict.bin"; + const std::string path = "crafted_binary_dict.bin"; FILE* fp = fopen(path.c_str(), "wb"); - fwrite(&numItems, sizeof(size_t), 1, fp); - fwrite(&keyTotalLength, sizeof(size_t), 1, fp); + const auto writeField = [fp](uint64_t value) { + FIELD field = static_cast(value); + fwrite(&field, sizeof(field), 1, fp); + }; + writeField(numItems); + writeField(keyTotalLength); fwrite(keyBuffer.data(), sizeof(char), keyBuffer.size(), fp); - fwrite(&valueTotalLength, sizeof(size_t), 1, fp); + writeField(valueTotalLength); fwrite(valueBuffer.data(), sizeof(char), valueBuffer.size(), fp); for (const auto& [numValues, keyOffset, valueOffsets] : items) { - fwrite(&numValues, sizeof(size_t), 1, fp); - fwrite(&keyOffset, sizeof(size_t), 1, fp); - for (size_t vo : valueOffsets) { - fwrite(&vo, sizeof(size_t), 1, fp); + writeField(numValues); + writeField(keyOffset); + for (uint64_t vo : valueOffsets) { + writeField(vo); } } fclose(fp); @@ -75,28 +83,84 @@ TEST_F(BinaryDictTest, Deserialization) { TestDict(deserializedTextDict); } +// The serialized layout must not depend on the word size of the platform: +// all integer fields are fixed 4-byte, so the total file size is fully +// determined by the lexicon content (#1412 follow-up). +TEST_F(BinaryDictTest, SerializedLayoutIsWordSizeIndependent) { + binDict->opencc::SerializableDict::SerializeToFile(fileName); + + FILE* fp = fopen(fileName.c_str(), "rb"); + ASSERT_NE(fp, nullptr); + fseek(fp, 0L, SEEK_END); + const size_t fileSize = static_cast(ftell(fp)); + fseek(fp, 0L, SEEK_SET); + uint32_t numItems; + ASSERT_EQ(fread(&numItems, sizeof(numItems), 1, fp), 1U); + fclose(fp); + + EXPECT_EQ(numItems, binDict->GetLexicon()->Length()); + + size_t expectedSize = 3 * sizeof(uint32_t); + for (const auto& entry : *binDict->GetLexicon()) { + expectedSize += entry->Key().length() + 1; // key + NUL + expectedSize += 2 * sizeof(uint32_t); // numValues + keyOffset + expectedSize += entry->NumValues() * sizeof(uint32_t); // value offsets + for (const std::string& value : entry->Values()) { + expectedSize += value.length() + 1; // value + NUL + } + } + EXPECT_EQ(fileSize, expectedSize); +} + +// An empty dictionary serializes to exactly 12 bytes and round-trips; this is +// the only fixed-width file whose first 8 bytes are all zero, which the +// legacy-layout detection must not mistake for a 64-bit file. +TEST_F(BinaryDictTest, EmptyDictRoundTrip) { + const std::string path = "empty_binary_dict.bin"; + const BinaryDictPtr emptyDict(new BinaryDict(LexiconPtr(new Lexicon))); + emptyDict->opencc::SerializableDict::SerializeToFile(path); + + FILE* fp = fopen(path.c_str(), "rb"); + ASSERT_NE(fp, nullptr); + fseek(fp, 0L, SEEK_END); + EXPECT_EQ(ftell(fp), static_cast(3 * sizeof(uint32_t))); + fclose(fp); + + const auto deserialized = SerializableDict::NewFromFile(path); + EXPECT_EQ(deserialized->GetLexicon()->Length(), 0U); + std::remove(path.c_str()); +} + // Test that keyTotalLength exceeding file size triggers InvalidFormat (#815). TEST_F(BinaryDictTest, RejectsHugeKeyTotalLength) { - std::string path = WriteMalformedBinaryDict( + std::string path = WriteCraftedBinaryDict( 1, 0x7000000000ULL, "", 0, "", {}); EXPECT_THROW(SerializableDict::NewFromFile(path), InvalidFormat); std::remove(path.c_str()); + + path = WriteCraftedBinaryDict(1, 0x70000000ULL, "", 0, "", {}); + EXPECT_THROW(SerializableDict::NewFromFile(path), InvalidFormat); + std::remove(path.c_str()); } // Test that valueTotalLength exceeding file size triggers InvalidFormat (#815). TEST_F(BinaryDictTest, RejectsHugeValueTotalLength) { std::string keyBuf = {'k', '\0'}; - std::string path = WriteMalformedBinaryDict( + std::string path = WriteCraftedBinaryDict( 1, 2, keyBuf, 0x7000000000ULL, "", {}); EXPECT_THROW(SerializableDict::NewFromFile(path), InvalidFormat); std::remove(path.c_str()); + + path = WriteCraftedBinaryDict(1, 2, keyBuf, 0x70000000ULL, "", {}); + EXPECT_THROW(SerializableDict::NewFromFile(path), InvalidFormat); + std::remove(path.c_str()); } // Test that a keyOffset beyond keyTotalLength triggers InvalidFormat (#813). TEST_F(BinaryDictTest, RejectsKeyOffsetOutOfBounds) { std::string keyBuf = {'h', 'i', '\0'}; std::string valBuf = {'v', '\0'}; - std::string path = WriteMalformedBinaryDict( + std::string path = WriteCraftedBinaryDict( 1, 3, keyBuf, 2, valBuf, {{1, 100, {0}}}); EXPECT_THROW(SerializableDict::NewFromFile(path), InvalidFormat); std::remove(path.c_str()); @@ -106,21 +170,34 @@ TEST_F(BinaryDictTest, RejectsKeyOffsetOutOfBounds) { TEST_F(BinaryDictTest, RejectsValueOffsetOutOfBounds) { std::string keyBuf = {'h', 'i', '\0'}; std::string valBuf = {'v', '\0'}; - std::string path = WriteMalformedBinaryDict( + std::string path = WriteCraftedBinaryDict( 1, 3, keyBuf, 2, valBuf, {{1, 0, {100}}}); EXPECT_THROW(SerializableDict::NewFromFile(path), InvalidFormat); std::remove(path.c_str()); } -// Sanity check: a well-formed BinaryDict crafted file deserializes. +// Sanity check: a well-formed fixed-width crafted file deserializes. TEST_F(BinaryDictTest, AcceptsWellFormedFile) { std::string keyBuf = {'h', 'i', '\0'}; std::string valBuf = {'v', '\0'}; - std::string path = WriteMalformedBinaryDict( + std::string path = WriteCraftedBinaryDict( 1, 3, keyBuf, 2, valBuf, {{1, 0, {0}}}); const auto deserialized = SerializableDict::NewFromFile(path); EXPECT_EQ(deserialized->GetLexicon()->Length(), 1); std::remove(path.c_str()); } +// Files written by old 64-bit builds use native 8-byte size_t fields and must +// still load. +TEST_F(BinaryDictTest, AcceptsLegacy64BitLayout) { + std::string keyBuf = {'h', 'i', '\0'}; + std::string valBuf = {'v', '\0'}; + std::string path = WriteCraftedBinaryDict( + 1, 3, keyBuf, 2, valBuf, {{1, 0, {0}}}); + const auto deserialized = SerializableDict::NewFromFile(path); + ASSERT_EQ(deserialized->GetLexicon()->Length(), 1); + EXPECT_EQ(deserialized->GetLexicon()->At(0)->Key(), "hi"); + std::remove(path.c_str()); +} + } // namespace opencc \ No newline at end of file diff --git a/src/DartsDict.cpp b/src/DartsDict.cpp index fbd60609d..e9a49e82c 100644 --- a/src/DartsDict.cpp +++ b/src/DartsDict.cpp @@ -234,13 +234,14 @@ DartsDictPtr DartsDict::NewFromFile(FILE* fp) { ? static_cast(fileEnd - currentOffset) : 0; - // Detect 32-bit vs 64-bit unit size by reading 8 bytes and checking + // Detect the dartsSize field width by reading 8 bytes and checking // whether bytes [4..7] are all zero. // - Old 64-bit build: dartsSize field is uint64_t (8 bytes); high 32 bits // are zero for any realistic file size → bytes [4..7] == 0. - // - 32-bit build (new or old): dartsSize field is uint32_t (4 bytes); - // bytes [4..7] are the first 4 bytes of the darts array (non-zero for - // any valid array whose root unit has a non-zero offset). + // - Current build (any word size) or old 32-bit build: dartsSize field is + // uint32_t (4 bytes); bytes [4..7] are the first 4 bytes of the darts + // array (non-zero for any valid array whose root unit has a non-zero + // offset). uint8_t probe[8]; if (fread(probe, 1, 8, fp) != 8) { throw InvalidFormat("Invalid OpenCC dictionary header (dartsSize)"); @@ -284,8 +285,8 @@ DartsDictPtr DartsDict::NewFromFile(FILE* fp) { return dict; } - // 32-bit: dartsSize is the first 4 bytes of probe; remaining 4 bytes belong - // to the array, so seek back. + // Fixed-width layout: dartsSize is the first 4 bytes of probe; remaining 4 + // bytes belong to the array, so seek back. fseek(fp, -4, SEEK_CUR); uint32_t dartsSize32; memcpy(&dartsSize32, probe, 4); @@ -376,8 +377,8 @@ DartsDictPtr DartsDict::NewFromBuffer(const char* data, size_t size) { return dict; } - // 32-bit: dartsSize is the first 4 bytes of probe; probe[4..7] belong to the - // darts array, so rewind 4 bytes before reading the array. + // Fixed-width layout: dartsSize is the first 4 bytes of probe; probe[4..7] + // belong to the darts array, so rewind 4 bytes before reading the array. uint32_t dartsSize32; memcpy(&dartsSize32, probe, 4); size_t dartsSize = dartsSize32; diff --git a/src/DartsDictTest.cpp b/src/DartsDictTest.cpp index 1d1e47b1a..897e56ed0 100644 --- a/src/DartsDictTest.cpp +++ b/src/DartsDictTest.cpp @@ -49,9 +49,10 @@ class DartsDictTest : public TextDictTestBase { } // Serialize dict as a legacy 64-bit OPENCCDARTS1 file by zero-extending - // each 32-bit unit to 64 bits and writing dartsSize as uint64_t. - // This reproduces files produced by old builds where id_type was size_t on - // 64-bit platforms. + // each 32-bit darts unit to 64 bits, writing dartsSize as uint64_t, and + // widening every BinaryDict integer field from the fixed 4-byte layout to + // 8 bytes. This reproduces files produced by old builds where id_type and + // size_t were 8 bytes on 64-bit platforms. static void SerializeAsLegacy64(const DartsDictPtr& dict, const std::string& path) { const std::string tmp = path + ".tmp"; @@ -65,7 +66,6 @@ class DartsDictTest : public TextDictTestBase { size_t numUnits = dartsSize32 / sizeof(uint32_t); std::vector units32(numUnits); fread(units32.data(), sizeof(uint32_t), numUnits, src); - long binaryStart = ftell(src); FILE* dst = fopen(path.c_str(), "wb"); const char* header = "OPENCCDARTS1"; @@ -78,12 +78,28 @@ class DartsDictTest : public TextDictTestBase { fwrite(&unit64, sizeof(uint64_t), 1, dst); } - // Copy BinaryDict section verbatim - fseek(src, binaryStart, SEEK_SET); - char buf[4096]; - size_t n; - while ((n = fread(buf, 1, sizeof(buf), src)) > 0) { - fwrite(buf, 1, n, dst); + // Widen the BinaryDict section field by field. + const auto widenField = [src, dst]() -> uint64_t { + uint32_t field32 = 0; + fread(&field32, sizeof(field32), 1, src); + uint64_t field64 = field32; + fwrite(&field64, sizeof(field64), 1, dst); + return field64; + }; + const auto copyBytes = [src, dst](uint64_t count) { + std::vector buf(static_cast(count)); + fread(buf.data(), sizeof(char), buf.size(), src); + fwrite(buf.data(), sizeof(char), buf.size(), dst); + }; + const uint64_t numItems = widenField(); + copyBytes(widenField()); // keyTotalLength + key buffer + copyBytes(widenField()); // valueTotalLength + value buffer + for (uint64_t i = 0; i < numItems; i++) { + const uint64_t numValues = widenField(); + widenField(); // keyOffset + for (uint64_t j = 0; j < numValues; j++) { + widenField(); // valueOffset + } } fclose(src); fclose(dst);