diff --git a/src/BUILD.bazel b/src/BUILD.bazel index 9156ad736..160ccb8af 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -222,6 +222,7 @@ cc_test( deps = [ ":conversion_lib", ":dict_group_test_base_lib", + ":marisa_dict_lib", "@googletest//:gtest_main", ], ) @@ -573,12 +574,16 @@ cc_library( cc_library( name = "prefix_match_lib", srcs = ["PrefixMatch.cpp"], - hdrs = ["PrefixMatch.hpp"], + hdrs = [ + "PrefixMatch.hpp", + "Utf8SkipScan.hpp", + ], deps = [ ":common_lib", ":dict_group_lib", ":dict_lib", ":lexicon_lib", + ":marisa_dict_lib", ":utf8_util_lib", ], ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7e99ab8ef..0cf4f3f81 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -56,6 +56,7 @@ set( SingleStageConverter.hpp StreamWindow.hpp UTF8StringSlice.hpp + Utf8SkipScan.hpp ) set( diff --git a/src/Conversion.cpp b/src/Conversion.cpp index 12fb61569..b1a21963a 100644 --- a/src/Conversion.cpp +++ b/src/Conversion.cpp @@ -51,6 +51,11 @@ void Conversion::AppendConverted(std::string_view phrase, if (matchedLength > remainingLength) { matchedLength = remainingLength; } + // Extend the unmatched run over characters that cannot begin any + // dictionary key; each of them would fail the prefix lookup, so they + // are consumed with a single bulk scan and one append. + matchedLength += prefixMatch->SkipUnmatchable( + pstr + matchedLength, remainingLength - matchedLength); output->append(pstr, matchedLength); } else { matchedLength = matched.keyLength; diff --git a/src/ConversionTest.cpp b/src/ConversionTest.cpp index 1b2a568d1..f4f74b42d 100644 --- a/src/ConversionTest.cpp +++ b/src/ConversionTest.cpp @@ -16,11 +16,55 @@ * limitations under the License. */ +#include + #include "Conversion.hpp" #include "DictGroupTestBase.hpp" +#include "MarisaDict.hpp" +#include "PrefixMatch.hpp" +#include "UTF8Util.hpp" namespace opencc { +namespace { + +// Reference implementation of the conversion loop that advances one +// character (or IDS sequence) per unmatched position and never bulk-skips. +// This is the pre-skip-scan behavior; Conversion::Convert must stay +// byte-for-byte equivalent to it on any input, including malformed UTF-8. +std::string ReferenceConvert(const PrefixMatch& prefixMatch, + const std::string& input) { + std::string output; + const char* pstr = input.data(); + const char* end = pstr + input.size(); + while (pstr < end) { + const size_t remaining = end - pstr; + const PrefixMatchView matched = prefixMatch.MatchPrefixView(pstr, remaining); + size_t matchedLength; + if (!matched.matched) { + matchedLength = + UTF8Util::NextIdeographicDescriptionSequenceLength(pstr, remaining); + if (matchedLength == 0) { + matchedLength = UTF8Util::NextCharLength(pstr); + } + if (matchedLength > remaining) { + matchedLength = remaining; + } + output.append(pstr, matchedLength); + } else { + matchedLength = matched.keyLength; + if (matchedLength > remaining) { + matchedLength = remaining; + } + output.append(matched.value.data(), matched.value.size()); + } + pstr += matchedLength; + } + return output; +} + +} // namespace + class ConversionTest : public DictGroupTestBase { protected: ConversionTest() @@ -47,6 +91,17 @@ TEST_F(ConversionTest, ConvertCString) { EXPECT_EQ(expected, converted); } +TEST_F(ConversionTest, ConvertMixedAsciiAndChinese) { + // Long ASCII runs exercise the bulk skip of unmatchable characters. + const std::string asciiPrefix = + "The quick brown fox jumps over the lazy dog 0123456789 "; + const std::string asciiSuffix = " tail"; + const std::string punctuation = utf8("、「」"); + EXPECT_EQ(asciiPrefix + expected + punctuation + expected + asciiSuffix, + conversion->Convert(asciiPrefix + input + punctuation + input + + asciiSuffix)); +} + TEST_F(ConversionTest, PreserveDictGroupPriority) { LexiconPtr firstLexicon(new Lexicon); firstLexicon->Add(DictEntryFactory::New("a", "X")); @@ -128,6 +183,78 @@ TEST_F(ConversionTest, DoesNotPreserveIsolatedIdeographicDescriptionCharacters) EXPECT_EQ(utf8("⿾证證"), idcDescriptionConversion.Convert(utf8("⿾证证"))); } +TEST_F(ConversionTest, DifferentialFuzzAgainstReferenceLoop) { + // Random inputs mixing dictionary keys, unrelated CJK, ASCII, IDS + // operators, and raw (possibly invalid or truncated) bytes. The skip-scan + // path must produce exactly what the per-character reference loop + // produces — same output or the same exception — on BOTH PrefixMatch + // constructions: the table path (dict group) and the single-dictionary + // fast path (MarisaDict answering prefix queries directly, whose skip + // table is built from a trie walk instead of materialized lexicons). + const MarisaDictPtr marisaDict = MarisaDict::NewFromDict(*dict); + const Conversion marisaConversion(marisaDict); + const PrefixMatch marisaReference(marisaDict); + PrefixMatch groupReference(dict); + + struct FuzzTarget { + const char* name; + const Conversion* conversion; + const PrefixMatch* reference; + }; + const std::vector targets = { + {"table-path", conversion.get(), &groupReference}, + {"fast-path", &marisaConversion, &marisaReference}, + }; + + const std::vector fragments = { + utf8("太后"), utf8("头发"), utf8("干燥"), utf8("鼠标"), + utf8("干"), utf8("后"), utf8("发"), utf8("里"), + utf8("天地"), utf8("、。"), utf8("⿰"), utf8("⿳"), + utf8("——"), utf8("🎉"), "ascii text ", "12345", + }; + for (const FuzzTarget& target : targets) { + std::mt19937 rng(20260724); + for (int iteration = 0; iteration < 300; iteration++) { + std::string input; + const int pieces = 1 + static_cast(rng() % 40); + for (int i = 0; i < pieces; i++) { + const uint32_t choice = rng() % 10; + if (choice < 7) { + input += fragments[rng() % fragments.size()]; + } else if (choice < 9) { + // Raw bytes, frequently invalid or truncated UTF-8. + const int count = 1 + static_cast(rng() % 4); + for (int b = 0; b < count; b++) { + input.push_back(static_cast(rng() % 256)); + } + } else if (!input.empty()) { + // Truncate the tail mid-character. + input.resize(input.size() - 1); + } + } + bool referenceThrew = false; + bool convertThrew = false; + std::string expected, actual; + try { + expected = ReferenceConvert(*target.reference, input); + } catch (const InvalidUTF8&) { + referenceThrew = true; + } + try { + actual = target.conversion->Convert(input); + } catch (const InvalidUTF8&) { + convertThrew = true; + } + ASSERT_EQ(referenceThrew, convertThrew) + << target.name << " iteration " << iteration; + if (!referenceThrew) { + ASSERT_EQ(expected, actual) + << target.name << " iteration " << iteration; + } + } + } +} + TEST_F(ConversionTest, TruncatedUtf8Sequence) { // This test specifically triggers the information disclosure vulnerability // in the old code. The bug occurs when a string ends with an incomplete diff --git a/src/MarisaDict.cpp b/src/MarisaDict.cpp index 8bc096cbe..d04b8192b 100644 --- a/src/MarisaDict.cpp +++ b/src/MarisaDict.cpp @@ -258,6 +258,39 @@ void MarisaDict::SerializeToFile(FILE* fp) const { new SerializedValues(GetLexicon())); serialized_values->SerializeToFile(fp); } + +bool MarisaDict::EnumerateKeys( + const std::function& cb) const { + // Prefer the already-reconstructed lexicon (e.g. when a table-path + // PrefixMatch materialized it); fall back to walking the trie so lazy + // loading is never forced. Same convention as ReconstructLexicon(): + // lexicon is written once before the release store of the flag, so the + // acquire load makes the plain read safe. + if (lexiconReconstructed.load(std::memory_order_acquire) && + lexicon != nullptr) { + for (const std::unique_ptr& entry : *lexicon) { + const std::string& key = entry->Key(); + cb(key.data(), key.length()); + } + return true; + } + const marisa::Trie* trie = internal->marisa.get(); + if (trie == nullptr) { + return false; + } + try { + marisa::Agent agent; + agent.set_query(""); + while (trie->predictive_search(agent)) { + cb(agent.key().ptr(), agent.key().length()); + } + } catch (const marisa::Exception&) { + // The trie has not been built or mapped yet. + return false; + } + return true; +} + PrefixMatchView MarisaDict::MatchPrefixValue(const char* word, size_t len) const { const marisa::Trie* trie = internal->marisa.get(); diff --git a/src/MarisaDict.hpp b/src/MarisaDict.hpp index a1918f7ad..86b864f96 100644 --- a/src/MarisaDict.hpp +++ b/src/MarisaDict.hpp @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include "Common.hpp" @@ -55,6 +56,15 @@ class OPENCC_EXPORT MarisaDict : public Dict, public SerializableDict { virtual PrefixMatchView MatchPrefixValue(const char* word, size_t len) const override; + /** + * Enumerates keys by walking the trie directly, so it does not trigger + * lexicon reconstruction. Returns false when the trie is unavailable. + * Not part of the Dict interface (MarisaDict-specific, non-virtual) so + * adding it does not affect Dict's or MarisaDict's ABI; callers that need + * it across dictionary types dynamic_cast to MarisaDict. + */ + bool EnumerateKeys(const std::function& cb) const; + virtual void SerializeToFile(FILE* fp) const override; /** diff --git a/src/MaxMatchSegmentation.cpp b/src/MaxMatchSegmentation.cpp index 0a7a19625..4ef587fc5 100644 --- a/src/MaxMatchSegmentation.cpp +++ b/src/MaxMatchSegmentation.cpp @@ -57,6 +57,11 @@ SegmentsPtr SegmentText(const std::shared_ptr& prefixMatch, if (matchedLength > remainingLength) { matchedLength = remainingLength; } + // Extend the unmatched run over characters that cannot begin any + // dictionary key; each of them would fail the prefix lookup, so they + // are consumed with a single bulk scan. + matchedLength += prefixMatch->SkipUnmatchable( + pstr + matchedLength, remainingLength - matchedLength); segLength += matchedLength; } else { clearBuffer(); diff --git a/src/MaxMatchSegmentationTest.cpp b/src/MaxMatchSegmentationTest.cpp index bb1d6fa3d..81e60c85e 100644 --- a/src/MaxMatchSegmentationTest.cpp +++ b/src/MaxMatchSegmentationTest.cpp @@ -43,6 +43,19 @@ TEST_F(MaxMatchSegmentationTest, Segment) { EXPECT_EQ(utf8("干燥"), std::string(segments->At(3))); } +TEST_F(MaxMatchSegmentationTest, MixedAsciiAndChinese) { + // ASCII runs and non-candidate CJK punctuation are skipped in bulk but must + // still land in the same segments as the per-character path produced. + const auto& segments = + segmenter->Segment(utf8("Hello, world! 太后。头发 done")); + EXPECT_EQ(5, segments->Length()); + EXPECT_EQ(utf8("Hello, world! "), std::string(segments->At(0))); + EXPECT_EQ(utf8("太后"), std::string(segments->At(1))); + EXPECT_EQ(utf8("。"), std::string(segments->At(2))); + EXPECT_EQ(utf8("头发"), std::string(segments->At(3))); + EXPECT_EQ(utf8(" done"), std::string(segments->At(4))); +} + TEST_F(MaxMatchSegmentationTest, EmptyString) { const auto& segments = segmenter->Segment(""); EXPECT_EQ(0, segments->Length()); diff --git a/src/PrefixMatch.cpp b/src/PrefixMatch.cpp index e53e7cfde..52b911b94 100644 --- a/src/PrefixMatch.cpp +++ b/src/PrefixMatch.cpp @@ -20,7 +20,9 @@ #include "Dict.hpp" #include "DictGroup.hpp" #include "Lexicon.hpp" +#include "MarisaDict.hpp" #include "UTF8Util.hpp" +#include "Utf8SkipScan.hpp" #include #include @@ -54,6 +56,7 @@ class PrefixMatch::Tables { public: class Matcher; std::unique_ptr matcher; + internal::Utf8SkipTable skip; }; namespace { @@ -61,6 +64,9 @@ namespace { struct CacheEntry { std::vector> dicts; std::weak_ptr tables; + // Fast-path Tables carry no matcher; never hand one to the table path + // (and vice versa), independently of the cache-key encoding. + bool fastPath = false; }; bool SameOwner(const std::weak_ptr& cached, @@ -123,6 +129,102 @@ void Unreachable() { #endif } +// Marks a single key's first character in the skip table: the lead byte +// always, plus the exact first code point for 2- and 3-byte characters so +// scanning can filter at character granularity. Uses the same decoder as the +// scanner (internal::DecodeCodePoint23) so a key's first character always +// maps to the bit the scanner will test. A key whose first character is +// truncated or invalid cannot be represented as a code point; character +// filtering is then disabled for the whole table and lead-byte filtering +// (which stops at the marked lead) remains in effect. +void MarkKeyFirstChar(internal::Utf8SkipTable* table, const char* key, + size_t len) { + if (len == 0) { + return; + } + // key may come straight from a trie enumeration buffer (e.g. marisa's + // Agent) and is NOT NUL-terminated. NextCharLengthNoException() reads only + // the lead byte, and DecodeCodePoint23() reads at most charLength bytes, + // which is checked against len first — no read may pass key + len. + const unsigned char lead = static_cast(key[0]); + table->candidate[lead] = true; + if (lead < 0x80) { + return; + } + const size_t charLength = UTF8Util::NextCharLengthNoException(key); + if (charLength == 0) { + // Invalid lead byte: the scanner stops at charLength == 0 before any + // candidate check, so the lead-byte mark alone is sufficient and + // character-level filtering can stay enabled. + return; + } + if (charLength > len) { + // Truncated first character: the key can still match its raw byte + // prefix, but the scanner would decode the code point using the text's + // following bytes and could skip such a position. This cannot be + // represented at character granularity, so fall back to lead-byte + // filtering (the marked lead stops the scan). + table->DisableCharLevel(); + return; + } + if (charLength == 2 || charLength == 3) { + table->MarkCharCandidate(internal::DecodeCodePoint23(key, charLength)); + } + // 4-byte (and longer legacy) first characters rely on lead-byte filtering. +} + +// Recursively marks every key's first character into table: groups recurse +// into children via GetDictGroupItems() (part of the Dict interface, so no +// dynamic_cast needed); MarisaDict is special-cased with a dynamic_cast so +// its keys can be walked directly from the trie without forcing lexicon +// reconstruction; any other leaf dict enumerates through GetLexicon(), whose +// contract (like LeafMatcher::AddDict's) assumes a non-null result. The +// nullptr branch below is an untested defensive guard, kept for symmetry +// with MarisaDict::EnumerateKeys() returning false when its trie is +// unavailable; it cannot currently be exercised through PrefixMatch's public +// API without a Dict subclass that violates the GetLexicon() contract +// elsewhere too (BuildMatcher's LeafMatcher::AddDict dereferences it +// unconditionally, so a real nullptr would already crash matcher +// construction). +void CollectSkipTable(const DictPtr& dict, internal::Utf8SkipTable* table) { + const std::list* items = dict->GetDictGroupItems(); + if (items != nullptr) { + for (const DictPtr& child : *items) { + CollectSkipTable(child, table); + } + return; + } + const MarisaDict* marisaDict = dynamic_cast(dict.get()); + if (marisaDict != nullptr) { + const bool enumerated = + marisaDict->EnumerateKeys([table](const char* key, size_t len) { + MarkKeyFirstChar(table, key, len); + }); + if (!enumerated) { + table->MarkAllCandidates(); + } + return; + } + const LexiconPtr lexicon = dict->GetLexicon(); + if (lexicon == nullptr) { + table->MarkAllCandidates(); + return; + } + for (const std::unique_ptr& entry : *lexicon) { + const std::string& key = entry->Key(); + MarkKeyFirstChar(table, key.data(), key.length()); + } +} + +// Builds the skip table for dict by enumerating every key at character +// granularity; falls back to treating every byte as a candidate when any +// leaf's keys are unavailable. +void BuildSkipTable(const DictPtr& dict, internal::Utf8SkipTable* table) { + table->EnableCharLevel(); + CollectSkipTable(dict, table); + table->Finalize(); +} + } // namespace class PrefixMatch::Tables::Matcher { @@ -338,18 +440,25 @@ PrefixMatch::PrefixMatch(const DictPtr& dict) { } } - if (actualDict && actualDict->SupportsFastPrefixMatch()) { + const bool fastPath = actualDict && actualDict->SupportsFastPrefixMatch(); + if (fastPath) { singleDict = actualDict; - return; } + // On the fast path the Tables only carry the skip table (the dictionary + // answers prefix queries itself); the cache is shared with the table path, + // with a distinct key prefix because fast-path Tables hold no matcher. + const DictPtr& tableDict = fastPath ? actualDict : dict; static std::mutex cacheMutex; static std::unordered_map> cache; std::string cacheKey; - AppendCacheKey(dict, &cacheKey); + if (fastPath) { + cacheKey.push_back('F'); + } + AppendCacheKey(tableDict, &cacheKey); std::vector> leafDicts; - CollectLeafDicts(dict, &leafDicts); + CollectLeafDicts(tableDict, &leafDicts); { std::lock_guard lock(cacheMutex); @@ -357,7 +466,7 @@ PrefixMatch::PrefixMatch(const DictPtr& dict) { const auto cached = cache.find(cacheKey); if (cached != cache.end()) { for (const CacheEntry& entry : cached->second) { - if (SameDicts(entry, leafDicts)) { + if (entry.fastPath == fastPath && SameDicts(entry, leafDicts)) { tables = entry.tables.lock(); if (tables != nullptr) { return; @@ -368,7 +477,10 @@ PrefixMatch::PrefixMatch(const DictPtr& dict) { } std::shared_ptr built(new Tables); - built->matcher = BuildMatcher(dict); + if (!fastPath) { + built->matcher = BuildMatcher(tableDict); + } + BuildSkipTable(tableDict, &built->skip); std::lock_guard lock(cacheMutex); PruneExpiredPrefixMatchCache(&cache); @@ -377,7 +489,7 @@ PrefixMatch::PrefixMatch(const DictPtr& dict) { it != entries.end();) { if (HasExpiredDict(*it)) { it = entries.erase(it); - } else if (SameDicts(*it, leafDicts)) { + } else if (it->fastPath == fastPath && SameDicts(*it, leafDicts)) { tables = it->tables.lock(); if (tables != nullptr) { return; @@ -391,6 +503,7 @@ PrefixMatch::PrefixMatch(const DictPtr& dict) { CacheEntry entry; entry.dicts = std::move(leafDicts); entry.tables = tables; + entry.fastPath = fastPath; entries.push_back(std::move(entry)); } @@ -423,6 +536,10 @@ PrefixMatch::Match PrefixMatch::MatchPrefix(const char* word, return Match{false, 0, nullptr, nullptr}; } +size_t PrefixMatch::SkipUnmatchable(const char* word, size_t len) const { + return internal::SkipNonCandidateBytes(tables->skip, word, len); +} + PrefixMatchView PrefixMatch::MatchPrefixView(const char* word, size_t len) const { if (singleDict != nullptr) { diff --git a/src/PrefixMatch.hpp b/src/PrefixMatch.hpp index 577fe8408..95c7ce2ef 100644 --- a/src/PrefixMatch.hpp +++ b/src/PrefixMatch.hpp @@ -54,11 +54,26 @@ class OPENCC_EXPORT PrefixMatch { */ PrefixMatchView MatchPrefixView(const char* word, size_t len) const; + /** + * Returns the number of leading bytes of @p word that are guaranteed not + * to begin any dictionary key: whole UTF-8 characters whose lead byte + * starts no key (ideographic description operators excluded, so the + * caller's IDS grouping is preserved). Callers may consume the returned + * run without calling MatchPrefixView() at any position inside it. The + * scan is word-at-a-time for ASCII runs. Returns 0 when the next character + * requires a real lookup. + */ + size_t SkipUnmatchable(const char* word, size_t len) const; + private: static void AppendCacheKey(const DictPtr& dict, std::string* output); static void CollectLeafDicts(const DictPtr& dict, std::vector>* output); + // Always non-null after construction. On the single-dictionary fast path + // it carries only the skip table; on the table path it also owns the + // matcher. Kept opaque so this header stays free of the skip-scan header + // and of PrefixMatch layout changes when the skip table evolves. std::shared_ptr tables; DictPtr singleDict; }; diff --git a/src/PrefixMatchTest.cpp b/src/PrefixMatchTest.cpp index 38109caf8..000632d78 100644 --- a/src/PrefixMatchTest.cpp +++ b/src/PrefixMatchTest.cpp @@ -27,6 +27,7 @@ #include "MarisaDict.hpp" #include "TestUtilsUTF8.hpp" #include "TextDictTestBase.hpp" +#include "Utf8SkipScan.hpp" namespace opencc { @@ -221,4 +222,182 @@ TEST_F(PrefixMatchTest, UnionGroupPreservesNestedShortCircuitBoundary) { EXPECT_EQ("root-second", *m.value); } +TEST(Utf8SkipScanTest, AsciiRunLength) { + const std::string cjk = utf8("一"); + // Cover all vector-width boundaries and unaligned starting offsets. + for (size_t asciiLength = 0; asciiLength < 70; asciiLength++) { + const std::string text = std::string(asciiLength, 'a') + cjk; + for (size_t offset = 0; offset <= asciiLength; offset++) { + EXPECT_EQ(asciiLength - offset, + internal::AsciiRunLength(text.data() + offset, + text.size() - offset)); + } + } + const std::string pureAscii(100, 'x'); + EXPECT_EQ(pureAscii.size(), + internal::AsciiRunLength(pureAscii.data(), pureAscii.size())); + EXPECT_EQ(0u, internal::AsciiRunLength(cjk.data(), cjk.size())); + EXPECT_EQ(0u, internal::AsciiRunLength("", 0)); +} + +TEST_F(PrefixMatchTest, SkipUnmatchableStopsAtAsciiCandidates) { + // textDict contains ASCII keys ("BYVoid", "zigzagzig"), so the scalar + // ASCII path must stop at candidate bytes 'B' and 'z'. + PrefixMatch pm(marisaDict); + const std::string query = "abc BYVoid"; + EXPECT_EQ(4u, pm.SkipUnmatchable(query.data(), query.size())); + EXPECT_EQ(0u, pm.SkipUnmatchable(query.data() + 4, query.size() - 4)); + const std::string zQuery = "zoo"; + EXPECT_EQ(0u, pm.SkipUnmatchable(zQuery.data(), zQuery.size())); +} + +TEST_F(PrefixMatchTest, SkipNeverCrossesAnyKeyFirstChar) { + // Invariant: for every dictionary key, the skip scan must not consume the + // position where that key starts, on both the fast path and the table + // path. A false skip here would silently corrupt conversion output. + PrefixMatch pmFast(marisaDict); + std::list dictsMulti = {marisaDict, textDict}; + DictPtr dictGroupMulti(new DictGroup(dictsMulti)); + PrefixMatch pmTable(dictGroupMulti); + const LexiconPtr lexicon = textDict->GetLexicon(); + for (const std::unique_ptr& entry : *lexicon) { + const std::string& key = entry->Key(); + EXPECT_EQ(0u, pmFast.SkipUnmatchable(key.data(), key.length())) << key; + EXPECT_EQ(0u, pmTable.SkipUnmatchable(key.data(), key.length())) << key; + } +} + +class PrefixMatchSkipTest : public ::testing::Test { +protected: + PrefixMatchSkipTest() { + LexiconPtr lexicon(new Lexicon); + lexicon->Add(DictEntryFactory::New(utf8("太后"), utf8("太后"))); + lexicon->Add(DictEntryFactory::New(utf8("里"), utf8("裏"))); + lexicon->Sort(); + dict = TextDictPtr(new TextDict(lexicon)); + } + + TextDictPtr dict; +}; + +TEST_F(PrefixMatchSkipTest, SkipsLongAsciiRuns) { + // No ASCII key exists, so long ASCII runs take the word-at-a-time scan. + PrefixMatch pm(dict); + const std::string ascii(100, 'x'); + EXPECT_EQ(ascii.size(), pm.SkipUnmatchable(ascii.data(), ascii.size())); + const std::string mixed = ascii + utf8("太后"); + EXPECT_EQ(ascii.size(), pm.SkipUnmatchable(mixed.data(), mixed.size())); +} + +TEST_F(PrefixMatchSkipTest, SkipsNonCandidateCjkCharacters) { + // 、。「」 and kana share the lead byte 0xE3 which starts no key; they are + // skipped without a lookup. 太 (0xE5) is a candidate and stops the scan. + PrefixMatch pm(dict); + const std::string punct = utf8("、。「」あい"); + EXPECT_EQ(punct.size(), pm.SkipUnmatchable(punct.data(), punct.size())); + const std::string mixed = punct + utf8("太后"); + EXPECT_EQ(punct.size(), pm.SkipUnmatchable(mixed.data(), mixed.size())); + const std::string candidate = utf8("太x"); + EXPECT_EQ(0u, pm.SkipUnmatchable(candidate.data(), candidate.size())); +} + +TEST_F(PrefixMatchSkipTest, FourByteCharactersUseLeadByteFiltering) { + // No 4-byte key exists in dict, so lead byte 0xF0 is not a candidate and + // 4-byte characters are skipped whole. + PrefixMatch pm(dict); + const std::string emoji = utf8("🎉🚀x太后"); + EXPECT_EQ(9u, pm.SkipUnmatchable(emoji.data(), emoji.size())); + + // With a 4-byte key, its lead byte becomes a candidate; any 4-byte + // character sharing that lead byte conservatively stops the scan. + LexiconPtr lexicon(new Lexicon); + lexicon->Add(DictEntryFactory::New(utf8("𠀀"), "X")); + lexicon->Sort(); + DictPtr extDict(new TextDict(lexicon)); + PrefixMatch pmExt(extDict); + const std::string extB = utf8("𠀀"); + const std::string otherFourByte = utf8("🎉"); + EXPECT_EQ(0u, pmExt.SkipUnmatchable(extB.data(), extB.size())); + EXPECT_EQ(0u, pmExt.SkipUnmatchable(otherFourByte.data(), + otherFourByte.size())); +} + +TEST_F(PrefixMatchSkipTest, CharLevelSkipsSameLeadByteNonCandidates) { + // Table path uses character-granularity filtering: 天/地 share the lead + // byte 0xE5 with the key 太后 but are not candidate first characters, so + // they are skipped; 太 itself stops the scan. + PrefixMatch pm(dict); + const std::string query = utf8("天地太后"); + EXPECT_EQ(6u, pm.SkipUnmatchable(query.data(), query.size())); + // 黑 shares the lead byte 0xE9 with the key 里. + const std::string query2 = utf8("黑里"); + EXPECT_EQ(3u, pm.SkipUnmatchable(query2.data(), query2.size())); +} + +TEST_F(PrefixMatchSkipTest, CharLevelSkipsNonIdsPunctuationWithLead0xE2) { + // Em dash and curly quotes share the lead byte 0xE2 with the IDS + // operators but are not operators themselves; character-level filtering + // skips them while U+2FF0..U+2FFF still stop the scan. + PrefixMatch pm(dict); + const std::string dashes = utf8("——“”…"); + EXPECT_EQ(dashes.size(), pm.SkipUnmatchable(dashes.data(), dashes.size())); + const std::string ids = utf8("—⿰氵青"); + EXPECT_EQ(3u, pm.SkipUnmatchable(ids.data(), ids.size())); +} + +TEST_F(PrefixMatchSkipTest, CharLevelSkipsTwoByteCharacters) { + PrefixMatch pm(dict); + const std::string query = utf8("Ünïcodé tail 太"); + EXPECT_EQ(query.size() - 3, pm.SkipUnmatchable(query.data(), query.size())); +} + +TEST_F(PrefixMatchSkipTest, AllIdsOperatorsStopTheScan) { + // Pins Finalize() to the arity table: every recognized IDS operator must + // stop the scan even though none of them begins a dictionary key, so the + // conversion loop's IDS grouping still sees them. + PrefixMatch pm(dict); + for (uint32_t cp = UTF8Util::kFirstIdeographicDescriptionOperator; + cp <= UTF8Util::kLastIdeographicDescriptionOperator; cp++) { + if (UTF8Util::IdeographicDescriptionOperatorArity(cp) == 0) { + continue; + } + const char encoded[3] = { + static_cast(0xE0 | (cp >> 12)), + static_cast(0x80 | ((cp >> 6) & 0x3F)), + static_cast(0x80 | (cp & 0x3F)), + }; + EXPECT_EQ(0u, pm.SkipUnmatchable(encoded, 3)) << "U+" << std::hex << cp; + } +} + +TEST_F(PrefixMatchSkipTest, StopsAtIdeographicDescriptionOperator) { + // No key starts with 0xE2, but IDS operators (U+2FF0..U+2FFF) must still + // reach the conversion loop so IDS grouping is preserved. + PrefixMatch pm(dict); + const std::string query = "abc" + utf8("⿰氵青"); + EXPECT_EQ(3u, pm.SkipUnmatchable(query.data(), query.size())); + EXPECT_EQ(0u, pm.SkipUnmatchable(query.data() + 3, query.size() - 3)); +} + +TEST_F(PrefixMatchSkipTest, StopsAtInvalidOrTruncatedSequences) { + PrefixMatch pm(dict); + const std::string invalid = std::string("ab") + '\xFF'; + EXPECT_EQ(2u, pm.SkipUnmatchable(invalid.data(), invalid.size())); + // Truncated 3-byte sequence: lead byte expects more bytes than remain. + const std::string truncated = std::string("ab") + '\xE3'; + EXPECT_EQ(2u, pm.SkipUnmatchable(truncated.data(), truncated.size())); + EXPECT_EQ(0u, pm.SkipUnmatchable(invalid.data() + 2, 1)); +} + +TEST_F(PrefixMatchTest, SkipUnmatchableTablePath) { + // Multi-dict group exercises the table path; skip bytes are the union of + // all leaf dictionaries' key start bytes. + std::list dictsMulti = {marisaDict, textDict}; + DictPtr dictGroupMulti(new DictGroup(dictsMulti)); + PrefixMatch pm(dictGroupMulti); + const std::string query = utf8("abc 清華"); + EXPECT_EQ(4u, pm.SkipUnmatchable(query.data(), query.size())); + EXPECT_EQ(0u, pm.SkipUnmatchable(query.data() + 4, query.size() - 4)); +} + } // namespace opencc diff --git a/src/UTF8Util.hpp b/src/UTF8Util.hpp index 9411c7b3f..6e60cf0b0 100644 --- a/src/UTF8Util.hpp +++ b/src/UTF8Util.hpp @@ -112,6 +112,15 @@ class OPENCC_EXPORT UTF8Util { return str - PrevCharLength(str); } + /** + * Code point bounds of the ideographic description operators recognized + * by IdeographicDescriptionOperatorArity(). Skip-scan tables iterate this + * range to keep IDS handling consistent with the conversion loop; when + * extending the switch below (e.g. with U+31EF), update these bounds too. + */ + static const uint32_t kFirstIdeographicDescriptionOperator = 0x2FF0; + static const uint32_t kLastIdeographicDescriptionOperator = 0x2FFF; + static size_t IdeographicDescriptionOperatorArity(uint32_t codePoint) { switch (codePoint) { case 0x2FF2: diff --git a/src/Utf8SkipScan.hpp b/src/Utf8SkipScan.hpp new file mode 100644 index 000000000..4762cb586 --- /dev/null +++ b/src/Utf8SkipScan.hpp @@ -0,0 +1,214 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2026 Carbo Kuo and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "UTF8Util.hpp" + +namespace opencc { +namespace internal { + +/** + * Decodes the code point of a 2- or 3-byte UTF-8 sequence from raw bytes, + * without validating continuation bytes (same masking as + * UTF8Util::CodePointNoException). Both the skip-table builder and the + * scanner must use this one decoder so that a key's first character always + * maps to the same bitmap bit as the text it can match. + */ +inline uint32_t DecodeCodePoint23(const char* str, size_t charLength) { + const unsigned char lead = static_cast(str[0]); + if (charLength == 2) { + return ((lead & 0x1FU) << 6) | + (static_cast(str[1]) & 0x3FU); + } + return ((lead & 0x0FU) << 12) | + ((static_cast(str[1]) & 0x3FU) << 6) | + (static_cast(str[2]) & 0x3FU); +} + +/** + * Per-byte lookup table describing which byte values may begin a dictionary + * key. The conversion hot loop uses it to consume runs of characters that + * cannot possibly match any key without paying for a trie lookup per + * character, and to scan the common all-ASCII case a word at a time. + */ +struct Utf8SkipTable { + /** + * candidate[b] is true when byte value b may begin a dictionary key (or is + * the lead byte of an ideographic description operator), so the conversion + * loop must run a full prefix lookup at that position. + */ + bool candidate[256] = {}; + /** + * True when any ASCII byte value is a candidate; disables the bulk + * ASCII-run scan. + */ + bool asciiHasCandidates = false; + /** + * Optional character-level refinement: one bit per BMP code point + * (U+0000..U+FFFF, 8 KiB). When non-empty, 2- and 3-byte UTF-8 characters + * are filtered by their exact code point instead of their lead byte, so + * e.g. a rare CJK character sharing its lead byte with common dictionary + * keys can still be skipped. 4-byte characters keep lead-byte filtering. + * Character-level mode is on exactly when this vector is non-empty; there + * is deliberately no separate flag to keep in sync. + */ + std::vector bmpCandidates; + + bool CharLevel() const { return !bmpCandidates.empty(); } + + void EnableCharLevel() { bmpCandidates.assign(0x10000 / 64, 0); } + + /** + * Permanently falls back to lead-byte filtering, e.g. when a key's first + * character is truncated or invalid and cannot be represented as a code + * point. Byte-level candidates accumulated so far remain valid. + */ + void DisableCharLevel() { bmpCandidates.clear(); } + + void MarkCharCandidate(uint32_t codePoint) { + if (CharLevel() && codePoint < 0x10000) { + bmpCandidates[codePoint >> 6] |= uint64_t(1) << (codePoint & 63); + } + } + + bool IsCharCandidate(uint32_t codePoint) const { + assert(CharLevel()); + return (bmpCandidates[codePoint >> 6] >> (codePoint & 63)) & 1; + } + + void MarkAllCandidates() { + for (size_t b = 0; b < 256; b++) { + candidate[b] = true; + } + DisableCharLevel(); + } + + /** + * Must be called after candidate[] is filled and before the table is used. + * Ideographic description operators must always stop the scan: the + * conversion loop groups IDS sequences without consulting the dictionary, + * and skipping over an operator would lose that grouping. The operator + * set is defined by UTF8Util::IdeographicDescriptionOperatorArity() and + * bounded by UTF8Util::kFirst/kLastIdeographicDescriptionOperator. + */ + void Finalize() { + static_assert(UTF8Util::kFirstIdeographicDescriptionOperator >= 0x0800 && + UTF8Util::kLastIdeographicDescriptionOperator <= 0xFFFF, + "IDS operator range must lie in the 3-byte UTF-8 block"); + for (uint32_t cp = UTF8Util::kFirstIdeographicDescriptionOperator; + cp <= UTF8Util::kLastIdeographicDescriptionOperator; cp++) { + if (UTF8Util::IdeographicDescriptionOperatorArity(cp) == 0) { + continue; + } + // The operator range lies in the 3-byte UTF-8 block of the BMP. + candidate[0xE0 | (cp >> 12)] = true; + MarkCharCandidate(cp); + } + asciiHasCandidates = false; + for (size_t b = 0; b < 0x80; b++) { + if (candidate[b]) { + asciiHasCandidates = true; + break; + } + } + } +}; + +/** + * Returns the number of leading bytes of [str, str + len) with the high bit + * clear, i.e. the length of the leading ASCII run. + * + * Word-at-a-time (SWAR) scan in plain C++: no intrinsics, no architecture + * or endianness special cases. Hand-written SSE2/NEON/WASM-SIMD kernels + * were measured against this and won by at most 3% end to end (see the + * commit that removed them): the candidate table has already eliminated the + * per-character trie lookups that dominated, so widening the scan from 8 to + * 16 bytes per step no longer moves the total. + */ +inline size_t AsciiRunLength(const char* str, size_t len) { + size_t pos = 0; + for (; pos + sizeof(uint64_t) <= len; pos += sizeof(uint64_t)) { + uint64_t word; + std::memcpy(&word, str + pos, sizeof(word)); + if ((word & UINT64_C(0x8080808080808080)) != 0) { + break; + } + } + // Resolve the exact position within the word that stopped the loop (at + // most 8 iterations), and handle the trailing partial word. + for (; pos < len; pos++) { + if (static_cast(str[pos]) >= 0x80) { + break; + } + } + return pos; +} + +/** + * Returns the number of leading bytes of [str, str + len) that the conversion + * loop may consume without any dictionary lookup: whole UTF-8 characters + * whose lead byte is not a candidate key-start byte. Scanning stops at the + * first candidate lead byte and at invalid or truncated sequences (which the + * caller's per-character path must handle). + */ +inline size_t SkipNonCandidateBytes(const Utf8SkipTable& table, const char* str, + size_t len) { + size_t pos = 0; + while (pos < len) { + const unsigned char lead = static_cast(str[pos]); + if (lead < 0x80) { + if (!table.asciiHasCandidates) { + pos += AsciiRunLength(str + pos, len - pos); + continue; + } + if (table.candidate[lead]) { + break; + } + pos++; + continue; + } + const size_t charLength = UTF8Util::NextCharLengthNoException(str + pos); + if (charLength == 0 || charLength > len - pos) { + break; + } + if (table.CharLevel() && (charLength == 2 || charLength == 3)) { + // DecodeCodePoint23 does not validate continuation bytes; the table + // builder decodes key bytes with the same function, so a position is + // skipped only when its exact bytes cannot begin any key. Either way + // the scan consumes the same charLength bytes the per-character path + // would, so behavior stays equivalent even on malformed input. + if (table.IsCharCandidate(DecodeCodePoint23(str + pos, charLength))) { + break; + } + } else if (table.candidate[lead]) { + break; + } + pos += charLength; + } + return pos; +} + +} // namespace internal +} // namespace opencc