Skip unmatchable character runs in the conversion hot path

src/BUILD.bazel

@@ -222,6 +222,7 @@ cc_test(
     deps = [
         ":conversion_lib",
         ":dict_group_test_base_lib",
+        ":marisa_dict_lib",
         "@googletest//:gtest_main",
     ],
 )
@@ -573,12 +574,16 @@ cc_library(
 cc_library(
     name = "prefix_match_lib",
     srcs = ["PrefixMatch.cpp"],
-    hdrs = ["PrefixMatch.hpp"],
+    hdrs = [
+        "PrefixMatch.hpp",
+        "Utf8SkipScan.hpp",
+    ],
     deps = [
         ":common_lib",
         ":dict_group_lib",
         ":dict_lib",
         ":lexicon_lib",
+        ":marisa_dict_lib",
         ":utf8_util_lib",
     ],
 )

src/CMakeLists.txt

@@ -56,6 +56,7 @@ set(
   SingleStageConverter.hpp
   StreamWindow.hpp
   UTF8StringSlice.hpp
+  Utf8SkipScan.hpp
 )
 
 set(

src/Conversion.cpp

@@ -51,6 +51,11 @@ void Conversion::AppendConverted(std::string_view phrase,
       if (matchedLength > remainingLength) {
         matchedLength = remainingLength;
       }
+      // Extend the unmatched run over characters that cannot begin any
+      // dictionary key; each of them would fail the prefix lookup, so they
+      // are consumed with a single bulk scan and one append.
+      matchedLength += prefixMatch->SkipUnmatchable(
+          pstr + matchedLength, remainingLength - matchedLength);
       output->append(pstr, matchedLength);
     } else {
       matchedLength = matched.keyLength;

src/ConversionTest.cpp

@@ -16,11 +16,55 @@
  * limitations under the License.
  */
 
+#include <random>
+
 #include "Conversion.hpp"
 #include "DictGroupTestBase.hpp"
+#include "MarisaDict.hpp"
+#include "PrefixMatch.hpp"
+#include "UTF8Util.hpp"
 
 namespace opencc {
 
+namespace {
+
+// Reference implementation of the conversion loop that advances one
+// character (or IDS sequence) per unmatched position and never bulk-skips.
+// This is the pre-skip-scan behavior; Conversion::Convert must stay
+// byte-for-byte equivalent to it on any input, including malformed UTF-8.
+std::string ReferenceConvert(const PrefixMatch& prefixMatch,
+                             const std::string& input) {
+  std::string output;
+  const char* pstr = input.data();
+  const char* end = pstr + input.size();
+  while (pstr < end) {
+    const size_t remaining = end - pstr;
+    const PrefixMatchView matched = prefixMatch.MatchPrefixView(pstr, remaining);
+    size_t matchedLength;
+    if (!matched.matched) {
+      matchedLength =
+          UTF8Util::NextIdeographicDescriptionSequenceLength(pstr, remaining);
+      if (matchedLength == 0) {
+        matchedLength = UTF8Util::NextCharLength(pstr);
+      }
+      if (matchedLength > remaining) {
+        matchedLength = remaining;
+      }
+      output.append(pstr, matchedLength);
+    } else {
+      matchedLength = matched.keyLength;
+      if (matchedLength > remaining) {
+        matchedLength = remaining;
+      }
+      output.append(matched.value.data(), matched.value.size());
+    }
+    pstr += matchedLength;
+  }
+  return output;
+}
+
+} // namespace
+
 class ConversionTest : public DictGroupTestBase {
 protected:
   ConversionTest()
@@ -47,6 +91,17 @@ TEST_F(ConversionTest, ConvertCString) {
   EXPECT_EQ(expected, converted);
 }
 
+TEST_F(ConversionTest, ConvertMixedAsciiAndChinese) {
+  // Long ASCII runs exercise the bulk skip of unmatchable characters.
+  const std::string asciiPrefix =
+      "The quick brown fox jumps over the lazy dog 0123456789 ";
+  const std::string asciiSuffix = " <html attr=\"value\">tail</html>";
+  const std::string punctuation = utf8("、「」");
+  EXPECT_EQ(asciiPrefix + expected + punctuation + expected + asciiSuffix,
+            conversion->Convert(asciiPrefix + input + punctuation + input +
+                                asciiSuffix));
+}
+
 TEST_F(ConversionTest, PreserveDictGroupPriority) {
   LexiconPtr firstLexicon(new Lexicon);
   firstLexicon->Add(DictEntryFactory::New("a", "X"));
@@ -128,6 +183,78 @@ TEST_F(ConversionTest, DoesNotPreserveIsolatedIdeographicDescriptionCharacters)
   EXPECT_EQ(utf8("⿾证證"), idcDescriptionConversion.Convert(utf8("⿾证证")));
 }
 
+TEST_F(ConversionTest, DifferentialFuzzAgainstReferenceLoop) {
+  // Random inputs mixing dictionary keys, unrelated CJK, ASCII, IDS
+  // operators, and raw (possibly invalid or truncated) bytes. The skip-scan
+  // path must produce exactly what the per-character reference loop
+  // produces — same output or the same exception — on BOTH PrefixMatch
+  // constructions: the table path (dict group) and the single-dictionary
+  // fast path (MarisaDict answering prefix queries directly, whose skip
+  // table is built from a trie walk instead of materialized lexicons).
+  const MarisaDictPtr marisaDict = MarisaDict::NewFromDict(*dict);
+  const Conversion marisaConversion(marisaDict);
+  const PrefixMatch marisaReference(marisaDict);
+  PrefixMatch groupReference(dict);
+
+  struct FuzzTarget {
+    const char* name;
+    const Conversion* conversion;
+    const PrefixMatch* reference;
+  };
+  const std::vector<FuzzTarget> targets = {
+      {"table-path", conversion.get(), &groupReference},
+      {"fast-path", &marisaConversion, &marisaReference},
+  };
+
+  const std::vector<std::string> fragments = {
+      utf8("太后"),   utf8("头发"),  utf8("干燥"),  utf8("鼠标"),
+      utf8("干"),     utf8("后"),    utf8("发"),    utf8("里"),
+      utf8("天地"),   utf8("、。"), utf8("⿰"),    utf8("⿳"),
+      utf8("——"),   utf8("🎉"),    "ascii text ",  "12345",
+  };
+  for (const FuzzTarget& target : targets) {
+    std::mt19937 rng(20260724);
+    for (int iteration = 0; iteration < 300; iteration++) {
+      std::string input;
+      const int pieces = 1 + static_cast<int>(rng() % 40);
+      for (int i = 0; i < pieces; i++) {
+        const uint32_t choice = rng() % 10;
+        if (choice < 7) {
+          input += fragments[rng() % fragments.size()];
+        } else if (choice < 9) {
+          // Raw bytes, frequently invalid or truncated UTF-8.
+          const int count = 1 + static_cast<int>(rng() % 4);
+          for (int b = 0; b < count; b++) {
+            input.push_back(static_cast<char>(rng() % 256));
+          }
+        } else if (!input.empty()) {
+          // Truncate the tail mid-character.
+          input.resize(input.size() - 1);
+        }
+      }
+      bool referenceThrew = false;
+      bool convertThrew = false;
+      std::string expected, actual;
+      try {
+        expected = ReferenceConvert(*target.reference, input);
+      } catch (const InvalidUTF8&) {
+        referenceThrew = true;
+      }
+      try {
+        actual = target.conversion->Convert(input);
+      } catch (const InvalidUTF8&) {
+        convertThrew = true;
+      }
+      ASSERT_EQ(referenceThrew, convertThrew)
+          << target.name << " iteration " << iteration;
+      if (!referenceThrew) {
+        ASSERT_EQ(expected, actual)
+            << target.name << " iteration " << iteration;
+      }
+    }
+  }
+}
+
 TEST_F(ConversionTest, TruncatedUtf8Sequence) {
   // This test specifically triggers the information disclosure vulnerability
   // in the old code. The bug occurs when a string ends with an incomplete

src/MarisaDict.cpp

@@ -258,6 +258,39 @@ void MarisaDict::SerializeToFile(FILE* fp) const {
       new SerializedValues(GetLexicon()));
   serialized_values->SerializeToFile(fp);
 }
+
+bool MarisaDict::EnumerateKeys(
+    const std::function<void(const char*, size_t)>& cb) const {
+  // Prefer the already-reconstructed lexicon (e.g. when a table-path
+  // PrefixMatch materialized it); fall back to walking the trie so lazy
+  // loading is never forced. Same convention as ReconstructLexicon():
+  // lexicon is written once before the release store of the flag, so the
+  // acquire load makes the plain read safe.
+  if (lexiconReconstructed.load(std::memory_order_acquire) &&
+      lexicon != nullptr) {
+    for (const std::unique_ptr<DictEntry>& entry : *lexicon) {
+      const std::string& key = entry->Key();
+      cb(key.data(), key.length());
+    }
+    return true;
+  }
+  const marisa::Trie* trie = internal->marisa.get();
+  if (trie == nullptr) {
+    return false;
+  }
+  try {
+    marisa::Agent agent;
+    agent.set_query("");
+    while (trie->predictive_search(agent)) {
+      cb(agent.key().ptr(), agent.key().length());
+    }
+  } catch (const marisa::Exception&) {
+    // The trie has not been built or mapped yet.
+    return false;
+  }
+  return true;
+}
+
 PrefixMatchView MarisaDict::MatchPrefixValue(const char* word,
                                              size_t len) const {
   const marisa::Trie* trie = internal->marisa.get();

src/MarisaDict.hpp

@@ -19,6 +19,7 @@
 #pragma once
 
 #include <atomic>
+#include <functional>
 #include <mutex>
 
 #include "Common.hpp"
@@ -55,6 +56,15 @@ class OPENCC_EXPORT MarisaDict : public Dict, public SerializableDict {
   virtual PrefixMatchView MatchPrefixValue(const char* word,
                                            size_t len) const override;
 
+  /**
+   * Enumerates keys by walking the trie directly, so it does not trigger
+   * lexicon reconstruction. Returns false when the trie is unavailable.
+   * Not part of the Dict interface (MarisaDict-specific, non-virtual) so
+   * adding it does not affect Dict's or MarisaDict's ABI; callers that need
+   * it across dictionary types dynamic_cast to MarisaDict.
+   */
+  bool EnumerateKeys(const std::function<void(const char*, size_t)>& cb) const;
+
   virtual void SerializeToFile(FILE* fp) const override;
 
   /**

src/MaxMatchSegmentation.cpp

@@ -57,6 +57,11 @@ SegmentsPtr SegmentText(const std::shared_ptr<PrefixMatch>& prefixMatch,
       if (matchedLength > remainingLength) {
         matchedLength = remainingLength;
       }
+      // Extend the unmatched run over characters that cannot begin any
+      // dictionary key; each of them would fail the prefix lookup, so they
+      // are consumed with a single bulk scan.
+      matchedLength += prefixMatch->SkipUnmatchable(
+          pstr + matchedLength, remainingLength - matchedLength);
       segLength += matchedLength;
     } else {
       clearBuffer();

src/MaxMatchSegmentationTest.cpp

@@ -43,6 +43,19 @@ TEST_F(MaxMatchSegmentationTest, Segment) {
   EXPECT_EQ(utf8("干燥"), std::string(segments->At(3)));
 }
 
+TEST_F(MaxMatchSegmentationTest, MixedAsciiAndChinese) {
+  // ASCII runs and non-candidate CJK punctuation are skipped in bulk but must
+  // still land in the same segments as the per-character path produced.
+  const auto& segments =
+      segmenter->Segment(utf8("Hello, world! 太后。头发 done"));
+  EXPECT_EQ(5, segments->Length());
+  EXPECT_EQ(utf8("Hello, world! "), std::string(segments->At(0)));
+  EXPECT_EQ(utf8("太后"), std::string(segments->At(1)));
+  EXPECT_EQ(utf8("。"), std::string(segments->At(2)));
+  EXPECT_EQ(utf8("头发"), std::string(segments->At(3)));
+  EXPECT_EQ(utf8(" done"), std::string(segments->At(4)));
+}
+
 TEST_F(MaxMatchSegmentationTest, EmptyString) {
   const auto& segments = segmenter->Segment("");
   EXPECT_EQ(0, segments->Length());