diff --git a/MODULE.bazel b/MODULE.bazel index 7c94fe62c..fecf02f46 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -30,6 +30,11 @@ single_version_override( version = "2.0.3", ) +local_path_override( + module_name = "darts-clone", + path = "deps/darts-clone-0.32h", +) + # Windows x64 node.lib import library for Zig/MinGW addon linking. node-addon-api # comes from BCR (above) and the Node C headers come from the rules_nodejs Node # toolchain (below); this import library is available from neither. diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index d865717f0..eb4d55ff5 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -69,8 +69,6 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/source.json": "7ad77c1e8c1b84222d9b3f3cae016a76639435744c19330b0b37c0a3c9da7dc0", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", - "https://bcr.bazel.build/modules/darts-clone/0.32/MODULE.bazel": "bdd235e31dd7f2538ff8b3ab3ef09c831349b141afca587d32b487d75c502361", - "https://bcr.bazel.build/modules/darts-clone/0.32/source.json": "c65158c152e276f3c59dc0fc0fa746f1ff601e23b0a09812e024fe563e4dc99c", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/MODULE.bazel": "f1b7bb2dd53e8f2ef984b39485ec8a44e9076dda5c4b8efd2fb4c6a6e856a31d", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/source.json": "ebe931bfe362e4b41e59ee00a528db6074157ff2ced92eb9e970acab2e1089c9", diff --git a/deps/darts-clone-0.32h/BUILD.bazel b/deps/darts-clone-0.32h/BUILD.bazel new file mode 100644 index 000000000..80112720f --- /dev/null +++ b/deps/darts-clone-0.32h/BUILD.bazel @@ -0,0 +1,6 @@ +cc_library( + name = "darts-clone", + hdrs = glob(["include/*.h"]), + strip_include_prefix = "include", + visibility = ["//visibility:public"], +) diff --git a/deps/darts-clone-0.32h/COPYING.md b/deps/darts-clone-0.32h/COPYING.md new file mode 100644 index 000000000..43ed4225a --- /dev/null +++ b/deps/darts-clone-0.32h/COPYING.md @@ -0,0 +1,11 @@ +# The BSD 2-clause license + +Copyright (c) 2008-2014, Susumu Yata +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/deps/darts-clone-0.32h/MODULE.bazel b/deps/darts-clone-0.32h/MODULE.bazel new file mode 100644 index 000000000..6f44052b9 --- /dev/null +++ b/deps/darts-clone-0.32h/MODULE.bazel @@ -0,0 +1,7 @@ +"""Darts-clone is a clone of Darts (Double-ARray Trie System).""" + +module( + name = "darts-clone", + version = "0.32h", + compatibility_level = 0, +) diff --git a/deps/darts-clone-0.32/darts.h b/deps/darts-clone-0.32h/include/darts.h similarity index 92% rename from deps/darts-clone-0.32/darts.h rename to deps/darts-clone-0.32h/include/darts.h index d47b0e36d..0de6b1167 100644 --- a/deps/darts-clone-0.32/darts.h +++ b/deps/darts-clone-0.32h/include/darts.h @@ -2,6 +2,7 @@ #define DARTS_H_ #include +#include #include #include @@ -173,6 +174,14 @@ class DoubleArrayImpl { array_ = static_cast(ptr); size_ = size; } + void copy_array(const void* ptr, std::size_t num_bytes) { + clear(); + const std::size_t extra_padding = num_bytes % unit_size() == 0 ? 0 : 1; + size_ = num_bytes / unit_size() + extra_padding; + buf_ = new unit_type[size_]; + std::memcpy(buf_, ptr, num_bytes); + array_ = buf_; + } // array() returns a pointer to the array of units. const void *array() const { return array_; @@ -283,6 +292,24 @@ class DoubleArrayImpl { std::size_t max_num_results, std::size_t length = 0, std::size_t node_pos = 0) const; + // The 1st commonLongestPrefixSearch() earches for the longest key which + // matches a prefix of the given string, and if it exists, its value and + // length are set to `result'. Otherwise, the value and the length of + //`result' are set to -1 and 0 respectively. Note that if `length' is 0, + // `key' is handled as a zero-terminated string. `node_pos' works as well as + // in exactMatchSearch(). + template + void commonLongestPrefixSearch(const key_type *key, U &result, + std::size_t length = 0, std::size_t node_pos = 0) const { + result = commonLongestPrefixSearch(key, length, node_pos); + } + // The 2nd commonLongestPrefixSearch() returns a result instead of updating + // the 2nd argument. So, the following commonLongestPrefixSearch() has only + // 3 arguments. + template + inline U commonLongestPrefixSearch(const key_type *key, + std::size_t length = 0, std::size_t node_pos = 0) const; + // In Darts-clone, a dictionary is a deterministic finite-state automaton // (DFA) and traverse() tests transitions on the DFA. The initial state is // `node_pos' and traverse() chooses transitions labeled key[key_pos], @@ -297,6 +324,8 @@ class DoubleArrayImpl { inline value_type traverse(const key_type *key, std::size_t &node_pos, std::size_t &key_pos, std::size_t length = 0) const; + bool validate(value_type max_value_limit = -1) const; + private: typedef Details::uchar_type uchar_type; typedef Details::id_type id_type; @@ -346,24 +375,52 @@ int DoubleArrayImpl::open(const char *file_name, size = std::ftell(file) - offset; } + size /= unit_size(); + if (size < 256 || (size & 0xFF) != 0) { + std::fclose(file); + return -1; + } + if (std::fseek(file, offset, SEEK_SET) != 0) { std::fclose(file); return -1; } - size /= unit_size(); + unit_type units[256]; + if (std::fread(units, unit_size(), 256, file) != 256) { + std::fclose(file); + return -1; + } + + if (units[0].label() != '\0' || units[0].has_leaf() || + units[0].offset() == 0 || units[0].offset() >= 512) { + std::fclose(file); + return -1; + } + for (id_type i = 1; i < 256; ++i) { + if (units[i].label() <= 0xFF && units[i].offset() >= size) { + std::fclose(file); + return -1; + } + } + unit_type *buf; try { buf = new unit_type[size]; + for (id_type i = 0; i < 256; ++i) { + buf[i] = units[i]; + } } catch (const std::bad_alloc &) { std::fclose(file); DARTS_THROW("failed to open double-array: std::bad_alloc"); } - if (std::fread(buf, unit_size(), size, file) != size) { - std::fclose(file); - delete[] buf; - return -1; + if (size > 256) { + if (std::fread(buf + 256, unit_size(), size - 256, file) != size - 256) { + std::fclose(file); + delete[] buf; + return -1; + } } std::fclose(file); @@ -377,7 +434,7 @@ int DoubleArrayImpl::open(const char *file_name, template int DoubleArrayImpl::save(const char *file_name, - const char *mode, std::size_t) const { + const char *mode, std::size_t offset) const { if (size() == 0) { return -1; } @@ -394,6 +451,11 @@ int DoubleArrayImpl::save(const char *file_name, } #endif + if (std::fseek(file, offset, SEEK_SET) != 0) { + std::fclose(file); + return -1; + } + if (std::fwrite(array_, unit_size(), size(), file) != size()) { std::fclose(file); return -1; @@ -402,6 +464,29 @@ int DoubleArrayImpl::save(const char *file_name, return 0; } +template +bool DoubleArrayImpl::validate(value_type max_value_limit) const { + if (size_ == 0 || array_ == NULL) { + return false; + } + if (array_[0].label() != '\0' || array_[0].has_leaf() || + array_[0].offset() == 0 || ((0 ^ array_[0].offset()) | 0xFF) >= size_) { + return false; + } + for (std::size_t i = 1; i < size_; ++i) { + if (array_[i].label() <= 0xFF) { + if (((i ^ array_[i].offset()) | 0xFF) >= size_) { + return false; + } + } else if (max_value_limit >= 0) { + if (array_[i].value() >= max_value_limit) { + return false; + } + } + } + return true; +} + template template inline U DoubleArrayImpl::exactMatchSearch(const key_type *key, @@ -484,6 +569,49 @@ inline std::size_t DoubleArrayImpl::commonPrefixSearch( return num_results; } +template +template +inline U DoubleArrayImpl::commonLongestPrefixSearch( + const key_type *key, std::size_t length, + std::size_t node_pos) const { + U result; + set_result(&result, static_cast(-1), 0); + + unit_type unit = array_[node_pos]; + node_pos ^= unit.offset(); + if (length != 0) { + for (std::size_t i = 0; i < length; ++i) { + node_pos ^= static_cast(key[i]); + unit = array_[node_pos]; + if (unit.label() != static_cast(key[i])) { + return result; + } + + node_pos ^= unit.offset(); + if (unit.has_leaf()) { + set_result(&result, static_cast( + array_[node_pos].value()), i + 1); + } + } + } else { + for ( ; key[length] != '\0'; ++length) { + node_pos ^= static_cast(key[length]); + unit = array_[node_pos]; + if (unit.label() != static_cast(key[length])) { + return result; + } + + node_pos ^= unit.offset(); + if (unit.has_leaf()) { + set_result(&result, static_cast( + array_[node_pos].value()), length + 1); + } + } + } + + return result; +} + template inline typename DoubleArrayImpl::value_type DoubleArrayImpl::traverse(const key_type *key, @@ -861,7 +989,7 @@ class Keyset { bool has_values() const { return values_ != NULL; } - value_type values(std::size_t id) const { + const value_type values(std::size_t id) const { if (has_values()) { return static_cast(values_[id]); } diff --git a/package.json b/package.json index 42533a9ba..b9e5af944 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,8 @@ "data/scripts/reverse.py", "deps/marisa-0.3.1/include", "deps/marisa-0.3.1/lib", + "deps/darts-clone-0.32h/COPYING.md", + "deps/darts-clone-0.32h/include", "deps/rapidjson-1.1.0/rapidjson", "node/cli.js", "node/configs.gypi", diff --git a/scripts/build-cli-windows-zig.sh b/scripts/build-cli-windows-zig.sh index a12c116c4..0922f5648 100755 --- a/scripts/build-cli-windows-zig.sh +++ b/scripts/build-cli-windows-zig.sh @@ -116,7 +116,7 @@ set +e -Ideps/marisa-0.3.1/include \ -Ideps/marisa-0.3.1/lib \ -Ideps/tclap-1.2.5 \ - -Ideps/darts-clone-0.32 \ + -Ideps/darts-clone-0.32h/include \ "${sources[@]}" \ -lshell32 \ -Wl,--strip-all \ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a053e42ee..31dc8bc4d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -100,7 +100,7 @@ set(UNITTESTS if (ENABLE_DARTS) set(OPENCC_ENABLE_DARTS 1) if(NOT USE_SYSTEM_DARTS) - include_directories(SYSTEM ../deps/darts-clone-0.32) + include_directories(SYSTEM ../deps/darts-clone-0.32h/include) endif() set( LIBOPENCC_HEADERS diff --git a/src/DartsDict.cpp b/src/DartsDict.cpp index d3541fd9e..c76e15166 100644 --- a/src/DartsDict.cpp +++ b/src/DartsDict.cpp @@ -145,12 +145,15 @@ DartsDictPtr DartsDict::NewFromFile(FILE* fp) { throw InvalidFormat( "Invalid OpenCC dictionary (dartsSize exceeds file size)"); } + if (dartsSize % doubleArray->unit_size() != 0) { + throw InvalidFormat("Invalid OpenCC dictionary size of darts alignment"); + } buffer = malloc(dartsSize); bytesRead = fread(buffer, 1, dartsSize, fp); if (bytesRead != dartsSize) { throw InvalidFormat("Invalid OpenCC dictionary size of darts mismatch"); } - doubleArray->set_array(buffer); + doubleArray->set_array(buffer, dartsSize / doubleArray->unit_size()); auto internal = dict->internal; internal->buffer = buffer; @@ -158,6 +161,10 @@ DartsDictPtr DartsDict::NewFromFile(FILE* fp) { internal->doubleArray = doubleArray; dict->lexicon = internal->binary->GetLexicon(); dict->maxLength = internal->binary->KeyMaxLength(); + if (!doubleArray->validate(static_cast( + dict->lexicon->Length()))) { + throw InvalidFormat("Invalid OpenCC dictionary darts data"); + } return dict; } diff --git a/src/DartsDictTest.cpp b/src/DartsDictTest.cpp index b70624dde..2565c1b9f 100644 --- a/src/DartsDictTest.cpp +++ b/src/DartsDictTest.cpp @@ -39,6 +39,40 @@ class DartsDictTest : public TextDictTestBase { return path; } + static void CorruptDartsRootUnit(const std::string& path) { + FILE* fp = fopen(path.c_str(), "r+b"); + const char* header = "OPENCCDARTS1"; + fseek(fp, static_cast(strlen(header) + sizeof(size_t)), SEEK_SET); + size_t invalidRoot = 0; + fwrite(&invalidRoot, sizeof(size_t), 1, fp); + fclose(fp); + } + + static void CorruptFirstDartsValue(const std::string& path) { + FILE* fp = fopen(path.c_str(), "r+b"); + const char* header = "OPENCCDARTS1"; + fseek(fp, static_cast(strlen(header)), SEEK_SET); + size_t dartsSize = 0; + fread(&dartsSize, sizeof(size_t), 1, fp); + const long dartsOffset = + static_cast(strlen(header) + sizeof(size_t)); + const size_t numUnits = dartsSize / sizeof(size_t); + for (size_t i = 0; i < numUnits; i++) { + size_t unit = 0; + fseek(fp, dartsOffset + static_cast(i * sizeof(size_t)), SEEK_SET); + fread(&unit, sizeof(size_t), 1, fp); + const size_t label = unit & ((1ULL << 31) | 0xFF); + if (label > 0xFF) { + const size_t invalidValue = (1ULL << 31) | ((1ULL << 31) - 1); + fseek(fp, dartsOffset + static_cast(i * sizeof(size_t)), + SEEK_SET); + fwrite(&invalidValue, sizeof(size_t), 1, fp); + break; + } + } + fclose(fp); + } + const DartsDictPtr dartsDict; const std::string fileName; }; @@ -86,4 +120,33 @@ TEST_F(DartsDictTest, RejectsHugeDartsSize) { std::remove(path.c_str()); } +TEST_F(DartsDictTest, RejectsMisalignedDartsSize) { + std::string path = WriteMalformedDartsFile(1); + FILE* fp = fopen(path.c_str(), "ab"); + char padding = '\0'; + fwrite(&padding, sizeof(char), 1, fp); + fclose(fp); + + EXPECT_THROW(SerializableDict::NewFromFile(path), InvalidFormat); + std::remove(path.c_str()); +} + +TEST_F(DartsDictTest, RejectsInvalidDartsRoot) { + const std::string path = "invalid_darts_root.ocd"; + dartsDict->opencc::SerializableDict::SerializeToFile(path); + CorruptDartsRootUnit(path); + + EXPECT_THROW(SerializableDict::NewFromFile(path), InvalidFormat); + std::remove(path.c_str()); +} + +TEST_F(DartsDictTest, RejectsInvalidDartsValue) { + const std::string path = "invalid_darts_value.ocd"; + dartsDict->opencc::SerializableDict::SerializeToFile(path); + CorruptFirstDartsValue(path); + + EXPECT_THROW(SerializableDict::NewFromFile(path), InvalidFormat); + std::remove(path.c_str()); +} + } // namespace opencc