Declare phrase-level character mapping dependencies

CONTRIBUTING.md

@@ -15,6 +15,7 @@ OpenCC 以 [Apache License 2.0](LICENSE) 釋出。提交 Pull Request、issue/co
 - [撰寫測試案例](#撰寫測試案例)
 - [Jieba 插件測試](#jieba-插件測試)
 - [簡轉繁轉換的特殊注意事項](#簡轉繁轉換的特殊注意事項)
+- [字級規則與詞表依賴](#字級規則與詞表依賴)
 
 ## 新增詞典條目
 
@@ -72,6 +73,21 @@ OpenCC 以 [Apache License 2.0](LICENSE) 釋出。提交 Pull Request、issue/co
 2. 每行一個條目
 3. 檔案使用 UTF-8 編碼
 
+### 4. 檢查字級規則與詞表依賴
+
+詞組級詞表如果仍保留逐字的 `A -> B` 轉換，對應的字級詞表也必須顯式
+保留 `A` 的 `B` 候選。若一般情況下不應再把 `A` 預設轉為 `B`，請不要
+直接刪除字級關係；應改為非預設候選，例如 `A<TAB>A B`。這樣單字預設
+仍保留為 `A`，但詞表中的 `A -> B` 依賴不會變成隱式規則。
+
+修改 `STCharacters.txt`、`TSCharacters.txt`、`TWVariants.txt`、
+`HKVariants.txt` 或其對應詞組詞表時，請同步檢查詞組中逐字對齊的
+替換是否仍在字表候選中。可執行以下測試檢查這類字級依賴：
+
+```bash
+bazel test //data/dictionary:dictionary_phrase_character_dependency_test
+```
+
 ## 排序詞典
 
 **重要**：詞典檔案必須按字典序排序，否則測試會失敗。

data/dictionary/BUILD.bazel

@@ -122,6 +122,31 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "dictionary_phrase_character_dependency_test",
+    size = "small",
+    srcs = ["DictionaryPhraseCharacterDependencyTest.cpp"],
+    data = [
+        "HKVariantsPhrases.txt",
+        "HKVariantsRev.txt",
+        "HKVariantsRevPhrases.txt",
+        "HKVariants.txt",
+        "STCharacters.txt",
+        "STPhrases.txt",
+        "TSCharacters.txt",
+        "TSPhrases.txt",
+        "TWVariantsRev.txt",
+        "TWVariantsRevPhrases.txt",
+        "TWVariants.txt",
+        "TWVariantsPhrases.txt",
+    ],
+    deps = [
+        "//src:utf8_util_lib",
+        "@bazel_tools//tools/cpp/runfiles",
+        "@googletest//:gtest_main",
+    ],
+)
+
 cc_test(
     name = "dictionary_variant_rev_phrases_test",
     size = "small",
@@ -149,6 +174,7 @@ test_suite(
     ] + [
         ":dictionary_TWPhrases_reverse_mapping_test",
         ":dictionary_TWPhrases_segmentation_test",
+        ":dictionary_phrase_character_dependency_test",
         ":dictionary_variant_rev_phrases_test",
     ],
 )

data/dictionary/DictionaryPhraseCharacterDependencyTest.cpp

@@ -0,0 +1,179 @@
+/*
+ * Open Chinese Convert
+ *
+ * Validate that phrase-level character substitutions remain declared in the
+ * corresponding character dictionary.
+ */
+
+#include "gtest/gtest.h"
+
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "src/UTF8Util.hpp"
+#include "tools/cpp/runfiles/runfiles.h"
+
+using bazel::tools::cpp::runfiles::Runfiles;
+
+namespace opencc {
+namespace {
+
+struct Entry {
+  size_t lineNumber;
+  std::string key;
+  std::vector<std::string> values;
+};
+
+struct DictionaryPair {
+  std::string phraseFile;
+  std::string characterFile;
+};
+
+std::vector<std::string> SplitValues(const std::string& values) {
+  std::vector<std::string> split;
+  size_t start = 0;
+  while (start <= values.size()) {
+    const size_t end = values.find(' ', start);
+    split.push_back(values.substr(start, end - start));
+    if (end == std::string::npos) {
+      break;
+    }
+    start = end + 1;
+  }
+  return split;
+}
+
+std::vector<std::string> SplitCharacters(const std::string& text) {
+  std::vector<std::string> characters;
+  const char* cursor = text.c_str();
+  while (*cursor != '\0') {
+    const size_t length = UTF8Util::NextCharLength(cursor);
+    characters.push_back(UTF8Util::FromSubstr(cursor, length));
+    cursor += length;
+  }
+  return characters;
+}
+
+bool IsDictionaryEntry(const std::string& line) {
+  return !line.empty() && line[0] != '#';
+}
+
+std::vector<Entry> LoadEntries(const std::string& path) {
+  std::ifstream stream(path);
+  EXPECT_TRUE(stream.is_open()) << path;
+
+  std::vector<Entry> entries;
+  std::string line;
+  size_t lineNumber = 0;
+  while (std::getline(stream, line)) {
+    ++lineNumber;
+    if (!IsDictionaryEntry(line)) {
+      continue;
+    }
+
+    const size_t tab = line.find('\t');
+    if (tab == std::string::npos) {
+      ADD_FAILURE() << path << ":" << lineNumber
+                    << " is missing a tab separator.";
+      continue;
+    }
+    entries.push_back(Entry{lineNumber, line.substr(0, tab),
+                            SplitValues(line.substr(tab + 1))});
+  }
+  return entries;
+}
+
+std::unordered_map<std::string, std::unordered_set<std::string>>
+LoadCharacterCandidates(const std::string& path) {
+  std::unordered_map<std::string, std::unordered_set<std::string>> candidates;
+  for (const Entry& entry : LoadEntries(path)) {
+    if (UTF8Util::Length(entry.key.c_str()) != 1) {
+      continue;
+    }
+    for (const std::string& value : entry.values) {
+      if (UTF8Util::Length(value.c_str()) == 1) {
+        candidates[entry.key].insert(value);
+      }
+    }
+  }
+  return candidates;
+}
+
+void ExpectPhraseCharacterSubstitutionsDeclared(
+    const Runfiles& runfiles, const DictionaryPair& dictionaryPair) {
+  const std::string dictionaryDir = "_main/data/dictionary/";
+  const std::string phrasePath =
+      runfiles.Rlocation(dictionaryDir + dictionaryPair.phraseFile);
+  const std::string characterPath =
+      runfiles.Rlocation(dictionaryDir + dictionaryPair.characterFile);
+
+  const std::vector<Entry> phraseEntries = LoadEntries(phrasePath);
+  const std::unordered_map<std::string, std::unordered_set<std::string>>
+      characterCandidates = LoadCharacterCandidates(characterPath);
+
+  std::ostringstream failures;
+  size_t failureCount = 0;
+  for (const Entry& entry : phraseEntries) {
+    const std::vector<std::string> keyCharacters = SplitCharacters(entry.key);
+    for (const std::string& value : entry.values) {
+      const std::vector<std::string> valueCharacters = SplitCharacters(value);
+      if (keyCharacters.size() != valueCharacters.size()) {
+        continue;
+      }
+
+      for (size_t i = 0; i < keyCharacters.size(); ++i) {
+        const std::string& source = keyCharacters[i];
+        const std::string& target = valueCharacters[i];
+        if (source == target) {
+          continue;
+        }
+
+        const auto candidates = characterCandidates.find(source);
+        if (candidates != characterCandidates.end() &&
+            candidates->second.count(target) != 0) {
+          continue;
+        }
+
+        ++failureCount;
+        failures << dictionaryPair.phraseFile << ":" << entry.lineNumber
+                 << " maps \"" << entry.key << "\" -> \"" << value
+                 << "\", including character substitution \"" << source
+                 << "\" -> \"" << target << "\", but "
+                 << dictionaryPair.characterFile
+                 << " does not list that target as a candidate.\n";
+      }
+    }
+  }
+
+  EXPECT_EQ(0U, failureCount)
+      << "Phrase-level character substitutions must also be declared in the "
+         "corresponding character dictionary. If a direct character conversion "
+         "is not generally desirable, keep it as a non-default candidate such "
+         "as A<TAB>A B instead of deleting it outright:\n"
+      << failures.str();
+}
+
+TEST(DictionaryPhraseCharacterDependencyTest,
+     PhraseSubstitutionsRemainDeclaredInCharacterDictionaries) {
+  std::unique_ptr<Runfiles> runfiles(Runfiles::CreateForTest());
+  ASSERT_NE(nullptr, runfiles);
+
+  const std::vector<DictionaryPair> dictionaryPairs = {
+      {"STPhrases.txt", "STCharacters.txt"},
+      {"TSPhrases.txt", "TSCharacters.txt"},
+      {"HKVariantsPhrases.txt", "HKVariants.txt"},
+      {"HKVariantsRevPhrases.txt", "HKVariantsRev.txt"},
+      {"TWVariantsPhrases.txt", "TWVariants.txt"},
+      {"TWVariantsRevPhrases.txt", "TWVariantsRev.txt"},
+  };
+  for (const DictionaryPair& dictionaryPair : dictionaryPairs) {
+    ExpectPhraseCharacterSubstitutionsDeclared(*runfiles, dictionaryPair);
+  }
+}
+
+} // namespace
+} // namespace opencc

data/dictionary/HKVariants.txt

@@ -41,6 +41,7 @@
 稅	税
 竈	灶
 糉	粽 糉 糭
+線	線 綫
 縕	緼
 纔	才
 脣	唇

data/dictionary/HKVariantsRevPhrases.txt

@@ -45,6 +45,7 @@
 丟巧針	丟巧針
 中才	中才
 中核	中核
+丹稜	丹稜
 之核	之核
 九針	九針
 乾冒煙	乾冒煙
@@ -781,6 +782,7 @@
 秀才造反	秀才造反
 秒針	秒針
 秧針	秧針
+穆稜	穆稜
 穿針	穿針
 穿針引線	穿針引線
 穿針走線	穿針走線
@@ -796,6 +798,7 @@
 絕才	絕才
 絞包針	絞包針
 給個棒錘當針認	給個棒錘當針認
+綏稜	綏稜
 經世之才	經世之才
 經國之才	經國之才
 經濟之才	經濟之才