Upgrade vendored darts-clone to v0.32h and harden OCD dictionary loading

MODULE.bazel

@@ -30,6 +30,11 @@ single_version_override(
     version = "2.0.3",
 )
 
+local_path_override(
+    module_name = "darts-clone",
+    path = "deps/darts-clone-0.32h",
+)
+
 # Windows x64 node.lib import library for Zig/MinGW addon linking. node-addon-api
 # comes from BCR (above) and the Node C headers come from the rules_nodejs Node
 # toolchain (below); this import library is available from neither.

deps/darts-clone-0.32h/BUILD.bazel

@@ -0,0 +1,6 @@
+cc_library(
+    name = "darts-clone",
+    hdrs = glob(["include/*.h"]),
+    strip_include_prefix = "include",
+    visibility = ["//visibility:public"],
+)

deps/darts-clone-0.32h/COPYING.md

@@ -0,0 +1,11 @@
+# The BSD 2-clause license
+
+Copyright (c) 2008-2014, Susumu Yata
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

deps/darts-clone-0.32h/MODULE.bazel

@@ -0,0 +1,7 @@
+"""Darts-clone is a clone of Darts (Double-ARray Trie System)."""
+
+module(
+    name = "darts-clone",
+    version = "0.32h",
+    compatibility_level = 0,
+)

deps/darts-clone-0.32/darts.h → deps/darts-clone-0.32h/include/darts.h

@@ -2,6 +2,7 @@
 #define DARTS_H_
 
 #include <cstdio>
+#include <cstring>
 #include <exception>
 #include <new>
 
@@ -173,6 +174,14 @@ class DoubleArrayImpl {
     array_ = static_cast<const unit_type *>(ptr);
     size_ = size;
   }
+  void copy_array(const void* ptr, std::size_t num_bytes) {
+    clear();
+    const std::size_t extra_padding = num_bytes % unit_size() == 0 ? 0 : 1;
+    size_ = num_bytes / unit_size() + extra_padding;
+    buf_ = new unit_type[size_];
+    std::memcpy(buf_, ptr, num_bytes);
+    array_ = buf_;
+  }
   // array() returns a pointer to the array of units.
   const void *array() const {
     return array_;
@@ -283,6 +292,24 @@ class DoubleArrayImpl {
       std::size_t max_num_results, std::size_t length = 0,
       std::size_t node_pos = 0) const;
 
+  // The 1st commonLongestPrefixSearch() earches for the longest key which
+  // matches a prefix of the given string, and if it exists, its value and
+  // length are set to `result'. Otherwise, the value and the length of
+  //`result' are set to -1 and 0 respectively. Note that if `length' is 0,
+  // `key' is handled as a zero-terminated string. `node_pos' works as well as
+  // in exactMatchSearch().
+  template <class U>
+  void commonLongestPrefixSearch(const key_type *key, U &result,
+      std::size_t length = 0, std::size_t node_pos = 0) const {
+    result = commonLongestPrefixSearch<U>(key, length, node_pos);
+  }
+  // The 2nd commonLongestPrefixSearch() returns a result instead of updating
+  // the 2nd argument. So, the following commonLongestPrefixSearch() has only
+  // 3 arguments.
+  template <class U>
+  inline U commonLongestPrefixSearch(const key_type *key,
+      std::size_t length = 0, std::size_t node_pos = 0) const;
+
   // In Darts-clone, a dictionary is a deterministic finite-state automaton
   // (DFA) and traverse() tests transitions on the DFA. The initial state is
   // `node_pos' and traverse() chooses transitions labeled key[key_pos],
@@ -297,6 +324,8 @@ class DoubleArrayImpl {
   inline value_type traverse(const key_type *key, std::size_t &node_pos,
       std::size_t &key_pos, std::size_t length = 0) const;
 
+  bool validate(value_type max_value_limit = -1) const;
+
  private:
   typedef Details::uchar_type uchar_type;
   typedef Details::id_type id_type;
@@ -346,24 +375,52 @@ int DoubleArrayImpl<A, B, T, C>::open(const char *file_name,
     size = std::ftell(file) - offset;
   }
 
+  size /= unit_size();
+  if (size < 256 || (size & 0xFF) != 0) {
+    std::fclose(file);
+    return -1;
+  }
+
   if (std::fseek(file, offset, SEEK_SET) != 0) {
     std::fclose(file);
     return -1;
   }
 
-  size /= unit_size();
+  unit_type units[256];
+  if (std::fread(units, unit_size(), 256, file) != 256) {
+    std::fclose(file);
+    return -1;
+  }
+
+  if (units[0].label() != '\0' || units[0].has_leaf() ||
+      units[0].offset() == 0 || units[0].offset() >= 512) {
+    std::fclose(file);
+    return -1;
+  }
+  for (id_type i = 1; i < 256; ++i) {
+    if (units[i].label() <= 0xFF && units[i].offset() >= size) {
+      std::fclose(file);
+      return -1;
+    }
+  }
+
   unit_type *buf;
   try {
     buf = new unit_type[size];
+    for (id_type i = 0; i < 256; ++i) {
+      buf[i] = units[i];
+    }
   } catch (const std::bad_alloc &) {
     std::fclose(file);
     DARTS_THROW("failed to open double-array: std::bad_alloc");
   }
 
-  if (std::fread(buf, unit_size(), size, file) != size) {
-    std::fclose(file);
-    delete[] buf;
-    return -1;
+  if (size > 256) {
+    if (std::fread(buf + 256, unit_size(), size - 256, file) != size - 256) {
+      std::fclose(file);
+      delete[] buf;
+      return -1;
+    }
   }
   std::fclose(file);
 
@@ -377,7 +434,7 @@ int DoubleArrayImpl<A, B, T, C>::open(const char *file_name,
 
 template <typename A, typename B, typename T, typename C>
 int DoubleArrayImpl<A, B, T, C>::save(const char *file_name,
-    const char *mode, std::size_t) const {
+    const char *mode, std::size_t offset) const {
   if (size() == 0) {
     return -1;
   }
@@ -394,6 +451,11 @@ int DoubleArrayImpl<A, B, T, C>::save(const char *file_name,
   }
 #endif
 
+  if (std::fseek(file, offset, SEEK_SET) != 0) {
+    std::fclose(file);
+    return -1;
+  }
+
   if (std::fwrite(array_, unit_size(), size(), file) != size()) {
     std::fclose(file);
     return -1;
@@ -402,6 +464,29 @@ int DoubleArrayImpl<A, B, T, C>::save(const char *file_name,
   return 0;
 }
 
+template <typename A, typename B, typename T, typename C>
+bool DoubleArrayImpl<A, B, T, C>::validate(value_type max_value_limit) const {
+  if (size_ == 0 || array_ == NULL) {
+    return false;
+  }
+  if (array_[0].label() != '\0' || array_[0].has_leaf() ||
+      array_[0].offset() == 0 || ((0 ^ array_[0].offset()) | 0xFF) >= size_) {
+    return false;
+  }
+  for (std::size_t i = 1; i < size_; ++i) {
+    if (array_[i].label() <= 0xFF) {
+      if (((i ^ array_[i].offset()) | 0xFF) >= size_) {
+        return false;
+      }
+    } else if (max_value_limit >= 0) {
+      if (array_[i].value() >= max_value_limit) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
 template <typename A, typename B, typename T, typename C>
 template <typename U>
 inline U DoubleArrayImpl<A, B, T, C>::exactMatchSearch(const key_type *key,
@@ -484,6 +569,49 @@ inline std::size_t DoubleArrayImpl<A, B, T, C>::commonPrefixSearch(
   return num_results;
 }
 
+template <typename A, typename B, typename T, typename C>
+template <typename U>
+inline U DoubleArrayImpl<A, B, T, C>::commonLongestPrefixSearch(
+    const key_type *key, std::size_t length,
+    std::size_t node_pos) const {
+  U result;
+  set_result(&result, static_cast<value_type>(-1), 0);
+
+  unit_type unit = array_[node_pos];
+  node_pos ^= unit.offset();
+  if (length != 0) {
+    for (std::size_t i = 0; i < length; ++i) {
+      node_pos ^= static_cast<uchar_type>(key[i]);
+      unit = array_[node_pos];
+      if (unit.label() != static_cast<uchar_type>(key[i])) {
+        return result;
+      }
+
+      node_pos ^= unit.offset();
+      if (unit.has_leaf()) {
+        set_result(&result, static_cast<value_type>(
+            array_[node_pos].value()), i + 1);
+      }
+    }
+  } else {
+    for ( ; key[length] != '\0'; ++length) {
+      node_pos ^= static_cast<uchar_type>(key[length]);
+      unit = array_[node_pos];
+      if (unit.label() != static_cast<uchar_type>(key[length])) {
+        return result;
+      }
+
+      node_pos ^= unit.offset();
+      if (unit.has_leaf()) {
+        set_result(&result, static_cast<value_type>(
+            array_[node_pos].value()), length + 1);
+      }
+    }
+  }
+
+  return result;
+}
+
 template <typename A, typename B, typename T, typename C>
 inline typename DoubleArrayImpl<A, B, T, C>::value_type
 DoubleArrayImpl<A, B, T, C>::traverse(const key_type *key,
@@ -861,7 +989,7 @@ class Keyset {
   bool has_values() const {
     return values_ != NULL;
   }
-  value_type values(std::size_t id) const {
+  const value_type values(std::size_t id) const {
     if (has_values()) {
       return static_cast<value_type>(values_[id]);
     }

package.json

@@ -44,6 +44,8 @@
     "data/scripts/reverse.py",
     "deps/marisa-0.3.1/include",
     "deps/marisa-0.3.1/lib",
+    "deps/darts-clone-0.32h/COPYING.md",
+    "deps/darts-clone-0.32h/include",
     "deps/rapidjson-1.1.0/rapidjson",
     "node/cli.js",
     "node/configs.gypi",

scripts/build-cli-windows-zig.sh

@@ -116,7 +116,7 @@ set +e
   -Ideps/marisa-0.3.1/include \
   -Ideps/marisa-0.3.1/lib \
   -Ideps/tclap-1.2.5 \
-  -Ideps/darts-clone-0.32 \
+  -Ideps/darts-clone-0.32h/include \
   "${sources[@]}" \
   -lshell32 \
   -Wl,--strip-all \

src/CMakeLists.txt

@@ -100,7 +100,7 @@ set(UNITTESTS
 if (ENABLE_DARTS)
   set(OPENCC_ENABLE_DARTS 1)
   if(NOT USE_SYSTEM_DARTS)
-    include_directories(SYSTEM ../deps/darts-clone-0.32)
+    include_directories(SYSTEM ../deps/darts-clone-0.32h/include)
   endif()
   set(
     LIBOPENCC_HEADERS

src/DartsDict.cpp

@@ -145,19 +145,26 @@ DartsDictPtr DartsDict::NewFromFile(FILE* fp) {
     throw InvalidFormat(
         "Invalid OpenCC dictionary (dartsSize exceeds file size)");
   }
+  if (dartsSize % doubleArray->unit_size() != 0) {
+    throw InvalidFormat("Invalid OpenCC dictionary size of darts alignment");
+  }
   buffer = malloc(dartsSize);
   bytesRead = fread(buffer, 1, dartsSize, fp);
   if (bytesRead != dartsSize) {
     throw InvalidFormat("Invalid OpenCC dictionary size of darts mismatch");
   }
-  doubleArray->set_array(buffer);
+  doubleArray->set_array(buffer, dartsSize / doubleArray->unit_size());
 
   auto internal = dict->internal;
   internal->buffer = buffer;
   internal->binary = BinaryDict::NewFromFile(fp);
   internal->doubleArray = doubleArray;
   dict->lexicon = internal->binary->GetLexicon();
   dict->maxLength = internal->binary->KeyMaxLength();
+  if (!doubleArray->validate(static_cast<Darts::DoubleArray::value_type>(
+          dict->lexicon->Length()))) {
+    throw InvalidFormat("Invalid OpenCC dictionary darts data");
+  }
   return dict;
 }