Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions src/BinaryDict.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,12 @@ uint64_t ReadField(const char* data, size_t size, size_t* offset,
// non-zero whenever the dictionary has entries, and an empty dictionary
// is exactly 12 bytes long.
// - Legacy 64-bit layout: the first field is an 8-byte numItems whose high
// 4 bytes are zero for any realistic dictionary.
// 32 bits are zero for any realistic dictionary.
size_t DetectFieldWidth(const char* data, size_t size) {
if (size < 8) {
return sizeof(uint32_t);
}
const bool highBytesZero =
data[4] == 0 && data[5] == 0 && data[6] == 0 && data[7] == 0;
if (highBytesZero && size != 3 * sizeof(uint32_t)) {
if (LooksLikeLegacy64Field(data) && size != 3 * sizeof(uint32_t)) {
return sizeof(uint64_t);
}
return sizeof(uint32_t);
Expand Down
18 changes: 18 additions & 0 deletions src/BinaryDict.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,28 @@

#pragma once

#include <cstdint>
#include <cstring>

#include "Common.hpp"
#include "SerializableDict.hpp"

namespace opencc {
// Returns true if the 8 bytes at `bytes` look like an 8-byte integer field
// from the legacy 64-bit OCD layout rather than two fixed-width uint32_t
// fields. Legacy files were written with native size_t fields in native byte
// order, so the probe is loaded as a native uint64_t: a legacy field has
// zero high 32 bits, while for a fixed-width file the high half is one of
// the two uint32 fields (or the start of the darts array), non-zero for any
// realistic dictionary. Works on both little- and big-endian hosts; legacy
// files themselves remain readable only on hosts of the byte order that
// wrote them.
inline bool LooksLikeLegacy64Field(const void* bytes) {
uint64_t word;
std::memcpy(&word, bytes, sizeof(word));
return (word >> 32) == 0;
}

/**
* Binary dictionary for faster deserialization
* @ingroup opencc_cpp_api
Expand Down
18 changes: 8 additions & 10 deletions src/DartsDict.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -234,20 +234,19 @@ DartsDictPtr DartsDict::NewFromFile(FILE* fp) {
? static_cast<size_t>(fileEnd - currentOffset)
: 0;

// Detect the dartsSize field width by reading 8 bytes and checking
// whether bytes [4..7] are all zero.
// Detect the dartsSize field width by reading 8 bytes.
// - Old 64-bit build: dartsSize field is uint64_t (8 bytes); high 32 bits
// are zero for any realistic file size → bytes [4..7] == 0.
// are zero for any realistic file size.
// - Current build (any word size) or old 32-bit build: dartsSize field is
// uint32_t (4 bytes); bytes [4..7] are the first 4 bytes of the darts
// array (non-zero for any valid array whose root unit has a non-zero
// offset).
// uint32_t (4 bytes); the other half of the probe is either the first
// 4 bytes of the darts array (non-zero for any valid array whose root
// unit has a non-zero offset) or dartsSize itself, so the native
// uint64_t load has non-zero high bits.
uint8_t probe[8];
if (fread(probe, 1, 8, fp) != 8) {
throw InvalidFormat("Invalid OpenCC dictionary header (dartsSize)");
}
bool is64bit =
(probe[4] == 0 && probe[5] == 0 && probe[6] == 0 && probe[7] == 0);
bool is64bit = LooksLikeLegacy64Field(probe);

if (is64bit) {
uint64_t dartsSize64;
Expand Down Expand Up @@ -340,8 +339,7 @@ DartsDictPtr DartsDict::NewFromBuffer(const char* data, size_t size) {
uint8_t probe[8];
memcpy(probe, data + offset, 8);
offset += 8;
bool is64bit =
(probe[4] == 0 && probe[5] == 0 && probe[6] == 0 && probe[7] == 0);
bool is64bit = LooksLikeLegacy64Field(probe);

if (is64bit) {
uint64_t dartsSize64;
Expand Down
Loading