Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ module(

bazel_dep(name = "aspect_bazel_lib", version = "2.22.5")
bazel_dep(name = "aspect_rules_js", version = "3.2.2")
bazel_dep(name = "darts-clone", version = "0.32")
bazel_dep(name = "darts-clone", version = "0.32h")
bazel_dep(name = "googletest", version = "1.17.0.bcr.2", dev_dependency = True)
bazel_dep(name = "marisa-trie", version = "0.3.1.bcr.2")
bazel_dep(name = "node_addon_api", version = "8.9.0")
Expand All @@ -30,9 +30,11 @@ single_version_override(
version = "2.0.3",
)

local_path_override(
single_version_override(
module_name = "darts-clone",
path = "deps/darts-clone-0.32h",
patches = ["//patches:darts-clone-opencc.patch"],
patch_strip = 1,
version = "0.32h",
)

# Windows x64 node.lib import library for Zig/MinGW addon linking. node-addon-api
Expand Down
38 changes: 38 additions & 0 deletions MODULE.bazel.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion deps/darts-clone-0.32h/MODULE.bazel
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Darts-clone is a clone of Darts (Double-ARray Trie System)."""
"""Darts-clone is a clone of Darts (Double-ARray Trie System) which is a C++ header library for a static double-array trie structure."""

module(
name = "darts-clone",
Expand Down
21 changes: 10 additions & 11 deletions deps/darts-clone-0.32h/include/darts.h
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#ifndef DARTS_H_
#define DARTS_H_

#include <cstdint>
#include <cstdio>
#include <cstring>
#include <exception>
Expand Down Expand Up @@ -38,7 +37,7 @@ typedef int value_type;

// The main structure of Darts-clone is an array of <DoubleArrayUnit>s, and the
// unit type is actually a wrapper of <id_type>.
typedef uint32_t id_type;
typedef unsigned int id_type;

// <progress_func_type> is the type of callback functions for reporting the
// progress of building a dictionary. See also build() of <DoubleArray>.
Expand Down Expand Up @@ -945,7 +944,7 @@ inline void BitVector::build() {

num_ones_ = 0;
for (std::size_t i = 0; i < units_.size(); ++i) {
ranks_[i] = num_ones_;
ranks_[i] = static_cast<id_type>(num_ones_);
num_ones_ += pop_count(units_[i]);
}
}
Expand Down Expand Up @@ -1867,7 +1866,7 @@ id_type DoubleArrayBuilder::arrange_from_keyset(const Keyset<T> &keyset,

inline id_type DoubleArrayBuilder::find_valid_offset(id_type id) const {
if (extras_head_ >= units_.size()) {
return units_.size() | (id & LOWER_MASK);
return static_cast<id_type>(units_.size()) | (id & LOWER_MASK);
}

id_type unfixed_id = extras_head_;
Expand All @@ -1879,7 +1878,7 @@ inline id_type DoubleArrayBuilder::find_valid_offset(id_type id) const {
unfixed_id = extras(unfixed_id).next();
} while (unfixed_id != extras_head_);

return units_.size() | (id & LOWER_MASK);
return static_cast<id_type>(units_.size()) | (id & LOWER_MASK);
}

inline bool DoubleArrayBuilder::is_valid_offset(id_type id,
Expand Down Expand Up @@ -1910,7 +1909,7 @@ inline void DoubleArrayBuilder::reserve_id(id_type id) {
if (id == extras_head_) {
extras_head_ = extras(id).next();
if (extras_head_ == id) {
extras_head_ = units_.size();
extras_head_ = static_cast<id_type>(units_.size());
}
}
extras(extras(id).prev()).set_next(extras(id).next());
Expand All @@ -1919,8 +1918,8 @@ inline void DoubleArrayBuilder::reserve_id(id_type id) {
}

inline void DoubleArrayBuilder::expand_units() {
id_type src_num_units = units_.size();
id_type src_num_blocks = num_blocks();
id_type src_num_units = static_cast<id_type>(units_.size());
id_type src_num_blocks = static_cast<id_type>(num_blocks());

id_type dest_num_units = src_num_units + BLOCK_SIZE;
id_type dest_num_blocks = src_num_blocks + 1;
Expand All @@ -1932,7 +1931,7 @@ inline void DoubleArrayBuilder::expand_units() {
units_.resize(dest_num_units);

if (dest_num_blocks > NUM_EXTRA_BLOCKS) {
for (std::size_t id = src_num_units; id < dest_num_units; ++id) {
for (id_type id = src_num_units; id < dest_num_units; ++id) {
extras(id).set_is_used(false);
extras(id).set_is_fixed(false);
}
Expand All @@ -1956,9 +1955,9 @@ inline void DoubleArrayBuilder::expand_units() {
inline void DoubleArrayBuilder::fix_all_blocks() {
id_type begin = 0;
if (num_blocks() > NUM_EXTRA_BLOCKS) {
begin = num_blocks() - NUM_EXTRA_BLOCKS;
begin = static_cast<id_type>(num_blocks()) - NUM_EXTRA_BLOCKS;
}
id_type end = num_blocks();
id_type end = static_cast<id_type>(num_blocks());

for (id_type block_id = begin; block_id != end; ++block_id) {
fix_block(block_id);
Expand Down
142 changes: 142 additions & 0 deletions patches/darts-clone-opencc.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
diff --git a/MODULE.bazel b/MODULE.bazel
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -6,4 +6,4 @@
compatibility_level = 0,
)

-bazel_dep(name = "rules_cc", version = "0.2.17")
+bazel_dep(name = "rules_cc", version = "0.2.20")
diff --git a/include/darts.h b/include/darts.h
--- a/include/darts.h
+++ b/include/darts.h
@@ -2,6 +2,7 @@
#define DARTS_H_

#include <cstdio>
+#include <cstring>
#include <exception>
#include <new>

@@ -173,6 +174,14 @@
array_ = static_cast<const unit_type *>(ptr);
size_ = size;
}
+ void copy_array(const void* ptr, std::size_t num_bytes) {
+ clear();
+ const std::size_t extra_padding = num_bytes % unit_size() == 0 ? 0 : 1;
+ size_ = num_bytes / unit_size() + extra_padding;
+ buf_ = new unit_type[size_];
+ std::memcpy(buf_, ptr, num_bytes);
+ array_ = buf_;
+ }
// array() returns a pointer to the array of units.
const void *array() const {
return array_;
@@ -314,6 +323,8 @@
// updates `node_pos' and `key_pos' after each transition.
inline value_type traverse(const key_type *key, std::size_t &node_pos,
std::size_t &key_pos, std::size_t length = 0) const;
+
+ bool validate(value_type max_value_limit = -1) const;

private:
typedef Details::uchar_type uchar_type;
@@ -451,6 +462,29 @@
}
std::fclose(file);
return 0;
+}
+
+template <typename A, typename B, typename T, typename C>
+bool DoubleArrayImpl<A, B, T, C>::validate(value_type max_value_limit) const {
+ if (size_ == 0 || array_ == NULL) {
+ return false;
+ }
+ if (array_[0].label() != '\0' || array_[0].has_leaf() ||
+ array_[0].offset() == 0 || ((0 ^ array_[0].offset()) | 0xFF) >= size_) {
+ return false;
+ }
+ for (std::size_t i = 1; i < size_; ++i) {
+ if (array_[i].label() <= 0xFF) {
+ if (((i ^ array_[i].offset()) | 0xFF) >= size_) {
+ return false;
+ }
+ } else if (max_value_limit >= 0) {
+ if (array_[i].value() >= max_value_limit) {
+ return false;
+ }
+ }
+ }
+ return true;
}

template <typename A, typename B, typename T, typename C>
@@ -910,7 +944,7 @@

num_ones_ = 0;
for (std::size_t i = 0; i < units_.size(); ++i) {
- ranks_[i] = num_ones_;
+ ranks_[i] = static_cast<id_type>(num_ones_);
num_ones_ += pop_count(units_[i]);
}
}
@@ -1832,7 +1866,7 @@

inline id_type DoubleArrayBuilder::find_valid_offset(id_type id) const {
if (extras_head_ >= units_.size()) {
- return units_.size() | (id & LOWER_MASK);
+ return static_cast<id_type>(units_.size()) | (id & LOWER_MASK);
}

id_type unfixed_id = extras_head_;
@@ -1844,7 +1878,7 @@
unfixed_id = extras(unfixed_id).next();
} while (unfixed_id != extras_head_);

- return units_.size() | (id & LOWER_MASK);
+ return static_cast<id_type>(units_.size()) | (id & LOWER_MASK);
}

inline bool DoubleArrayBuilder::is_valid_offset(id_type id,
@@ -1875,7 +1909,7 @@
if (id == extras_head_) {
extras_head_ = extras(id).next();
if (extras_head_ == id) {
- extras_head_ = units_.size();
+ extras_head_ = static_cast<id_type>(units_.size());
}
}
extras(extras(id).prev()).set_next(extras(id).next());
@@ -1884,8 +1918,8 @@
}

inline void DoubleArrayBuilder::expand_units() {
- id_type src_num_units = units_.size();
- id_type src_num_blocks = num_blocks();
+ id_type src_num_units = static_cast<id_type>(units_.size());
+ id_type src_num_blocks = static_cast<id_type>(num_blocks());

id_type dest_num_units = src_num_units + BLOCK_SIZE;
id_type dest_num_blocks = src_num_blocks + 1;
@@ -1897,7 +1931,7 @@
units_.resize(dest_num_units);

if (dest_num_blocks > NUM_EXTRA_BLOCKS) {
- for (std::size_t id = src_num_units; id < dest_num_units; ++id) {
+ for (id_type id = src_num_units; id < dest_num_units; ++id) {
extras(id).set_is_used(false);
extras(id).set_is_fixed(false);
}
@@ -1921,9 +1955,9 @@
inline void DoubleArrayBuilder::fix_all_blocks() {
id_type begin = 0;
if (num_blocks() > NUM_EXTRA_BLOCKS) {
- begin = num_blocks() - NUM_EXTRA_BLOCKS;
+ begin = static_cast<id_type>(num_blocks()) - NUM_EXTRA_BLOCKS;
}
- id_type end = num_blocks();
+ id_type end = static_cast<id_type>(num_blocks());

for (id_type block_id = begin; block_id != end; ++block_id) {
fix_block(block_id);
Loading