Skip to content

Serialize BinaryDict with fixed-width fields for word-size-independent OCD output #1414

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
frankslin merged 1 commit into from
Jul 9, 2026
Merged

Serialize BinaryDict with fixed-width fields for word-size-independent OCD output #1414

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 83 additions & 103 deletions src/BinaryDict.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstring>

#include "BinaryDict.hpp"
Expand All @@ -33,6 +34,58 @@ size_t BinaryDict::KeyMaxLength() const {
return maxLength;
}

namespace {

// All integer fields are serialized as fixed-width uint32_t so that the file
// is identical regardless of the word size of the platform that built it.
// Files written by old 64-bit builds used native 8-byte size_t fields; the
// deserializers below detect and accept both layouts.
void WriteField(FILE* fp, size_t value) {
if (value > UINT32_MAX) {
throw InvalidFormat("OpenCC binary dictionary field exceeds uint32 range");
}
uint32_t field = static_cast<uint32_t>(value);
fwrite(&field, sizeof(field), 1, fp);
}

uint64_t ReadField(const char* data, size_t size, size_t* offset,
size_t fieldWidth) {
if (fieldWidth > size || *offset > size - fieldWidth) {
throw InvalidFormat("Invalid OpenCC binary dictionary (truncated)");
}
uint64_t value;
if (fieldWidth == sizeof(uint32_t)) {
uint32_t field;
memcpy(&field, data + *offset, sizeof(field));
value = field;
} else {
memcpy(&value, data + *offset, sizeof(value));
}
*offset += fieldWidth;
return value;
}

// Detects the integer field width of a serialized BinaryDict.
// - Fixed-width layout (current builds and old 32-bit builds): the first 8
// bytes are numItems followed by keyTotalLength; keyTotalLength is
// non-zero whenever the dictionary has entries, and an empty dictionary
// is exactly 12 bytes long.
// - Legacy 64-bit layout: the first field is an 8-byte numItems whose high
// 4 bytes are zero for any realistic dictionary.
size_t DetectFieldWidth(const char* data, size_t size) {
if (size < 8) {
return sizeof(uint32_t);
}
const bool highBytesZero =
data[4] == 0 && data[5] == 0 && data[6] == 0 && data[7] == 0;
if (highBytesZero && size != 3 * sizeof(uint32_t)) {
return sizeof(uint64_t);
}
return sizeof(uint32_t);
}

} // namespace

void BinaryDict::SerializeToFile(FILE* fp) const {
std::string keyBuf, valueBuf;
std::vector<size_t> keyOffsets, valueOffsets;
Expand All @@ -41,26 +94,26 @@ void BinaryDict::SerializeToFile(FILE* fp) const {
valueTotalLength);
// Number of items
size_t numItems = lexicon->Length();
fwrite(&numItems, sizeof(size_t), 1, fp);
WriteField(fp, numItems);

// Data
fwrite(&keyTotalLength, sizeof(size_t), 1, fp);
WriteField(fp, keyTotalLength);
fwrite(keyBuf.c_str(), sizeof(char), keyTotalLength, fp);
fwrite(&valueTotalLength, sizeof(size_t), 1, fp);
WriteField(fp, valueTotalLength);
fwrite(valueBuf.c_str(), sizeof(char), valueTotalLength, fp);

size_t keyCursor = 0, valueCursor = 0;
for (const std::unique_ptr<DictEntry>& entry : *lexicon) {
// Number of values
size_t numValues = entry->NumValues();
fwrite(&numValues, sizeof(size_t), 1, fp);
WriteField(fp, numValues);
// Key offset
size_t keyOffset = keyOffsets[keyCursor++];
fwrite(&keyOffset, sizeof(size_t), 1, fp);
WriteField(fp, keyOffset);
// Values offset
for (size_t i = 0; i < numValues; i++) {
size_t valueOffset = valueOffsets[valueCursor++];
fwrite(&valueOffset, sizeof(size_t), 1, fp);
WriteField(fp, valueOffset);
}
}
assert(keyCursor == numItems);
Expand All @@ -69,120 +122,47 @@ void BinaryDict::SerializeToFile(FILE* fp) const {
BinaryDictPtr BinaryDict::NewFromFile(FILE* fp) {
long savedOffset = ftell(fp);
fseek(fp, 0L, SEEK_END);
long offsetBoundLong = ftell(fp) - savedOffset;
long remainingSizeLong = ftell(fp) - savedOffset;
fseek(fp, savedOffset, SEEK_SET);
assert(offsetBoundLong >= 0);
size_t offsetBound = static_cast<size_t>(offsetBoundLong);

BinaryDictPtr dict(new BinaryDict(LexiconPtr(new Lexicon)));

// Number of items
size_t numItems;
size_t unitsRead = fread(&numItems, sizeof(size_t), 1, fp);
if (unitsRead != 1) {
throw InvalidFormat("Invalid OpenCC binary dictionary (numItems)");
}

// Keys
size_t keyTotalLength;
unitsRead = fread(&keyTotalLength, sizeof(size_t), 1, fp);
if (unitsRead != 1) {
throw InvalidFormat("Invalid OpenCC binary dictionary (keyTotalLength)");
}
if (keyTotalLength > offsetBound) {
throw InvalidFormat("Invalid OpenCC binary dictionary (keyTotalLength exceeds file size)");
}
dict->keyBuffer.resize(keyTotalLength);
unitsRead = fread(const_cast<char*>(dict->keyBuffer.c_str()), sizeof(char),
keyTotalLength, fp);
if (unitsRead != keyTotalLength) {
throw InvalidFormat("Invalid OpenCC binary dictionary (keyBuffer)");
}

// Values
size_t valueTotalLength;
unitsRead = fread(&valueTotalLength, sizeof(size_t), 1, fp);
if (unitsRead != 1) {
throw InvalidFormat("Invalid OpenCC binary dictionary (valueTotalLength)");
}
if (valueTotalLength > offsetBound) {
throw InvalidFormat(
"Invalid OpenCC binary dictionary (valueTotalLength exceeds file size)");
}
dict->valueBuffer.resize(valueTotalLength);
unitsRead = fread(const_cast<char*>(dict->valueBuffer.c_str()), sizeof(char),
valueTotalLength, fp);
if (unitsRead != valueTotalLength) {
throw InvalidFormat("Invalid OpenCC binary dictionary (valueBuffer)");
}
assert(remainingSizeLong >= 0);
size_t remainingSize = static_cast<size_t>(remainingSizeLong);

// Offsets
for (size_t i = 0; i < numItems; i++) {
// Number of values
size_t numValues;
unitsRead = fread(&numValues, sizeof(size_t), 1, fp);
if (unitsRead != 1) {
throw InvalidFormat("Invalid OpenCC binary dictionary (numValues)");
}
// Key offset
size_t keyOffset;
unitsRead = fread(&keyOffset, sizeof(size_t), 1, fp);
if (unitsRead != 1 || keyOffset >= keyTotalLength) {
throw InvalidFormat("Invalid OpenCC binary dictionary (keyOffset)");
}
const char* keyStart = dict->keyBuffer.c_str() + keyOffset;
if (memchr(keyStart, '\0', keyTotalLength - keyOffset) == nullptr) {
throw InvalidFormat(
"Invalid OpenCC binary dictionary (key not null-terminated)");
}
std::string key = keyStart;
// Value offset
std::vector<std::string> values;
for (size_t j = 0; j < numValues; j++) {
size_t valueOffset;
unitsRead = fread(&valueOffset, sizeof(size_t), 1, fp);
if (unitsRead != 1 || valueOffset >= valueTotalLength) {
throw InvalidFormat("Invalid OpenCC binary dictionary (valueOffset)");
}
const char* valueStart = dict->valueBuffer.c_str() + valueOffset;
if (memchr(valueStart, '\0', valueTotalLength - valueOffset) == nullptr) {
throw InvalidFormat(
"Invalid OpenCC binary dictionary (value not null-terminated)");
}
values.push_back(valueStart);
}
DictEntry* entry = DictEntryFactory::New(key, values);
dict->lexicon->Add(entry);
std::string buffer;
buffer.resize(remainingSize);
size_t bytesRead = fread(const_cast<char*>(buffer.c_str()), sizeof(char),
remainingSize, fp);
if (bytesRead != remainingSize) {
throw InvalidFormat("Invalid OpenCC binary dictionary (truncated)");
}

return dict;
return NewFromBuffer(buffer.data(), buffer.size());
}

BinaryDictPtr BinaryDict::NewFromBuffer(const char* data, size_t size) {
BinaryDictPtr dict(new BinaryDict(LexiconPtr(new Lexicon)));
size_t offset = 0;
const size_t fieldWidth = DetectFieldWidth(data, size);

auto readSizeT = [&]() -> size_t {
if (offset + sizeof(size_t) > size) {
throw InvalidFormat("Invalid OpenCC binary dictionary (truncated)");
auto readField = [&]() -> size_t {
uint64_t value = ReadField(data, size, &offset, fieldWidth);
if (value > size) {
// Every field is a count, length or offset bounded by the data size;
// this also guarantees the value fits in size_t on 32-bit platforms.
throw InvalidFormat("Invalid OpenCC binary dictionary (field too large)");
}
size_t val;
memcpy(&val, data + offset, sizeof(size_t));
offset += sizeof(size_t);
return val;
return static_cast<size_t>(value);
};

size_t numItems = readSizeT();
size_t numItems = readField();

size_t keyTotalLength = readSizeT();
size_t keyTotalLength = readField();
if (keyTotalLength > size - offset) {
throw InvalidFormat(
"Invalid OpenCC binary dictionary (keyTotalLength exceeds data size)");
}
dict->keyBuffer.assign(data + offset, keyTotalLength);
offset += keyTotalLength;

size_t valueTotalLength = readSizeT();
size_t valueTotalLength = readField();
if (valueTotalLength > size - offset) {
throw InvalidFormat(
"Invalid OpenCC binary dictionary (valueTotalLength exceeds data size)");
Expand All @@ -191,8 +171,8 @@ BinaryDictPtr BinaryDict::NewFromBuffer(const char* data, size_t size) {
offset += valueTotalLength;

for (size_t i = 0; i < numItems; i++) {
size_t numValues = readSizeT();
size_t keyOffset = readSizeT();
size_t numValues = readField();
size_t keyOffset = readField();
if (keyOffset >= keyTotalLength) {
throw InvalidFormat("Invalid OpenCC binary dictionary (keyOffset)");
}
Expand All @@ -204,7 +184,7 @@ BinaryDictPtr BinaryDict::NewFromBuffer(const char* data, size_t size) {
std::string key = keyStart;
std::vector<std::string> values;
for (size_t j = 0; j < numValues; j++) {
size_t valueOffset = readSizeT();
size_t valueOffset = readField();
if (valueOffset >= valueTotalLength) {
throw InvalidFormat("Invalid OpenCC binary dictionary (valueOffset)");
}
Expand Down
Loading
Loading