From f12d9e8698e15c604b8e916b2d06da99f301087f Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Fri, 5 Jun 2026 08:28:30 -0700 Subject: [PATCH] Declare phrase-level character mapping dependencies Phrase dictionaries can continue to convert a character A to B even after the corresponding character-level A-to-B rule has been removed. That makes the remaining phrase entries an implicit dependency and can hide the fact that A-to-B is still supported in specific words. Add a dictionary test that compares equal-length phrase mappings with their corresponding character dictionaries. Whenever a phrase maps A to B at the same character position, the character dictionary must still list B as a candidate for A. If B should no longer be the default, the dictionary should keep it as a non-default candidate such as AA B rather than deleting it outright. Backfill the currently implicit ST, TS, and HK character candidates and add the reverse variant identity exceptions required by the expanded character candidate sets. Document the maintenance rule in CONTRIBUTING.md. --- CONTRIBUTING.md | 16 ++ data/dictionary/BUILD.bazel | 26 +++ ...ictionaryPhraseCharacterDependencyTest.cpp | 179 ++++++++++++++++++ data/dictionary/HKVariants.txt | 1 + data/dictionary/HKVariantsRevPhrases.txt | 3 + data/dictionary/STCharacters.txt | 49 ++++- data/dictionary/TSCharacters.txt | 5 + data/dictionary/TWVariantsRevPhrases.txt | 10 + 8 files changed, 279 insertions(+), 10 deletions(-) create mode 100644 data/dictionary/DictionaryPhraseCharacterDependencyTest.cpp diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8d6b2f222..dedf3f3fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,7 @@ OpenCC 以 [Apache License 2.0](LICENSE) 釋出。提交 Pull Request、issue/co - [撰寫測試案例](#撰寫測試案例) - [Jieba 插件測試](#jieba-插件測試) - [簡轉繁轉換的特殊注意事項](#簡轉繁轉換的特殊注意事項) +- [字級規則與詞表依賴](#字級規則與詞表依賴) ## 新增詞典條目 @@ -72,6 +73,21 @@ OpenCC 以 [Apache License 2.0](LICENSE) 釋出。提交 Pull Request、issue/co 2. 每行一個條目 3. 檔案使用 UTF-8 編碼 +### 4. 檢查字級規則與詞表依賴 + +詞組級詞表如果仍保留逐字的 `A -> B` 轉換，對應的字級詞表也必須顯式 +保留 `A` 的 `B` 候選。若一般情況下不應再把 `A` 預設轉為 `B`，請不要 +直接刪除字級關係；應改為非預設候選，例如 `AA B`。這樣單字預設 +仍保留為 `A`，但詞表中的 `A -> B` 依賴不會變成隱式規則。 + +修改 `STCharacters.txt`、`TSCharacters.txt`、`TWVariants.txt`、 +`HKVariants.txt` 或其對應詞組詞表時，請同步檢查詞組中逐字對齊的 +替換是否仍在字表候選中。可執行以下測試檢查這類字級依賴： + +```bash +bazel test //data/dictionary:dictionary_phrase_character_dependency_test +``` + ## 排序詞典 **重要**：詞典檔案必須按字典序排序，否則測試會失敗。 diff --git a/data/dictionary/BUILD.bazel b/data/dictionary/BUILD.bazel index c564107a7..4521135f0 100644 --- a/data/dictionary/BUILD.bazel +++ b/data/dictionary/BUILD.bazel @@ -122,6 +122,31 @@ cc_test( ], ) +cc_test( + name = "dictionary_phrase_character_dependency_test", + size = "small", + srcs = ["DictionaryPhraseCharacterDependencyTest.cpp"], + data = [ + "HKVariantsPhrases.txt", + "HKVariantsRev.txt", + "HKVariantsRevPhrases.txt", + "HKVariants.txt", + "STCharacters.txt", + "STPhrases.txt", + "TSCharacters.txt", + "TSPhrases.txt", + "TWVariantsRev.txt", + "TWVariantsRevPhrases.txt", + "TWVariants.txt", + "TWVariantsPhrases.txt", + ], + deps = [ + "//src:utf8_util_lib", + "@bazel_tools//tools/cpp/runfiles", + "@googletest//:gtest_main", + ], +) + cc_test( name = "dictionary_variant_rev_phrases_test", size = "small", @@ -149,6 +174,7 @@ test_suite( ] + [ ":dictionary_TWPhrases_reverse_mapping_test", ":dictionary_TWPhrases_segmentation_test", + ":dictionary_phrase_character_dependency_test", ":dictionary_variant_rev_phrases_test", ], ) diff --git a/data/dictionary/DictionaryPhraseCharacterDependencyTest.cpp b/data/dictionary/DictionaryPhraseCharacterDependencyTest.cpp new file mode 100644 index 000000000..16ad06089 --- /dev/null +++ b/data/dictionary/DictionaryPhraseCharacterDependencyTest.cpp @@ -0,0 +1,179 @@ +/* + * Open Chinese Convert + * + * Validate that phrase-level character substitutions remain declared in the + * corresponding character dictionary. + */ + +#include "gtest/gtest.h" + +#include +#include +#include +#include +#include +#include + +#include "src/UTF8Util.hpp" +#include "tools/cpp/runfiles/runfiles.h" + +using bazel::tools::cpp::runfiles::Runfiles; + +namespace opencc { +namespace { + +struct Entry { + size_t lineNumber; + std::string key; + std::vector values; +}; + +struct DictionaryPair { + std::string phraseFile; + std::string characterFile; +}; + +std::vector SplitValues(const std::string& values) { + std::vector split; + size_t start = 0; + while (start <= values.size()) { + const size_t end = values.find(' ', start); + split.push_back(values.substr(start, end - start)); + if (end == std::string::npos) { + break; + } + start = end + 1; + } + return split; +} + +std::vector SplitCharacters(const std::string& text) { + std::vector characters; + const char* cursor = text.c_str(); + while (*cursor != '\0') { + const size_t length = UTF8Util::NextCharLength(cursor); + characters.push_back(UTF8Util::FromSubstr(cursor, length)); + cursor += length; + } + return characters; +} + +bool IsDictionaryEntry(const std::string& line) { + return !line.empty() && line[0] != '#'; +} + +std::vector LoadEntries(const std::string& path) { + std::ifstream stream(path); + EXPECT_TRUE(stream.is_open()) << path; + + std::vector entries; + std::string line; + size_t lineNumber = 0; + while (std::getline(stream, line)) { + ++lineNumber; + if (!IsDictionaryEntry(line)) { + continue; + } + + const size_t tab = line.find('\t'); + if (tab == std::string::npos) { + ADD_FAILURE() << path << ":" << lineNumber + << " is missing a tab separator."; + continue; + } + entries.push_back(Entry{lineNumber, line.substr(0, tab), + SplitValues(line.substr(tab + 1))}); + } + return entries; +} + +std::unordered_map> +LoadCharacterCandidates(const std::string& path) { + std::unordered_map> candidates; + for (const Entry& entry : LoadEntries(path)) { + if (UTF8Util::Length(entry.key.c_str()) != 1) { + continue; + } + for (const std::string& value : entry.values) { + if (UTF8Util::Length(value.c_str()) == 1) { + candidates[entry.key].insert(value); + } + } + } + return candidates; +} + +void ExpectPhraseCharacterSubstitutionsDeclared( + const Runfiles& runfiles, const DictionaryPair& dictionaryPair) { + const std::string dictionaryDir = "_main/data/dictionary/"; + const std::string phrasePath = + runfiles.Rlocation(dictionaryDir + dictionaryPair.phraseFile); + const std::string characterPath = + runfiles.Rlocation(dictionaryDir + dictionaryPair.characterFile); + + const std::vector phraseEntries = LoadEntries(phrasePath); + const std::unordered_map> + characterCandidates = LoadCharacterCandidates(characterPath); + + std::ostringstream failures; + size_t failureCount = 0; + for (const Entry& entry : phraseEntries) { + const std::vector keyCharacters = SplitCharacters(entry.key); + for (const std::string& value : entry.values) { + const std::vector valueCharacters = SplitCharacters(value); + if (keyCharacters.size() != valueCharacters.size()) { + continue; + } + + for (size_t i = 0; i < keyCharacters.size(); ++i) { + const std::string& source = keyCharacters[i]; + const std::string& target = valueCharacters[i]; + if (source == target) { + continue; + } + + const auto candidates = characterCandidates.find(source); + if (candidates != characterCandidates.end() && + candidates->second.count(target) != 0) { + continue; + } + + ++failureCount; + failures << dictionaryPair.phraseFile << ":" << entry.lineNumber + << " maps \"" << entry.key << "\" -> \"" << value + << "\", including character substitution \"" << source + << "\" -> \"" << target << "\", but " + << dictionaryPair.characterFile + << " does not list that target as a candidate.\n"; + } + } + } + + EXPECT_EQ(0U, failureCount) + << "Phrase-level character substitutions must also be declared in the " + "corresponding character dictionary. If a direct character conversion " + "is not generally desirable, keep it as a non-default candidate such " + "as AA B instead of deleting it outright:\n" + << failures.str(); +} + +TEST(DictionaryPhraseCharacterDependencyTest, + PhraseSubstitutionsRemainDeclaredInCharacterDictionaries) { + std::unique_ptr runfiles(Runfiles::CreateForTest()); + ASSERT_NE(nullptr, runfiles); + + const std::vector dictionaryPairs = { + {"STPhrases.txt", "STCharacters.txt"}, + {"TSPhrases.txt", "TSCharacters.txt"}, + {"HKVariantsPhrases.txt", "HKVariants.txt"}, + {"HKVariantsRevPhrases.txt", "HKVariantsRev.txt"}, + {"TWVariantsPhrases.txt", "TWVariants.txt"}, + {"TWVariantsRevPhrases.txt", "TWVariantsRev.txt"}, + }; + for (const DictionaryPair& dictionaryPair : dictionaryPairs) { + ExpectPhraseCharacterSubstitutionsDeclared(*runfiles, dictionaryPair); + } +} + +} // namespace +} // namespace opencc diff --git a/data/dictionary/HKVariants.txt b/data/dictionary/HKVariants.txt index 0f722f7a5..f8b3ee5c4 100644 --- a/data/dictionary/HKVariants.txt +++ b/data/dictionary/HKVariants.txt @@ -41,6 +41,7 @@ 稅 税 竈 灶 糉 粽 糉 糭 +線 線 綫 縕 緼 纔 才 脣 唇 diff --git a/data/dictionary/HKVariantsRevPhrases.txt b/data/dictionary/HKVariantsRevPhrases.txt index a3eaf1123..e06ae5c2d 100644 --- a/data/dictionary/HKVariantsRevPhrases.txt +++ b/data/dictionary/HKVariantsRevPhrases.txt @@ -45,6 +45,7 @@ 丟巧針 丟巧針 中才 中才 中核 中核 +丹稜 丹稜 之核 之核 九針 九針 乾冒煙 乾冒煙 @@ -781,6 +782,7 @@ 秀才造反 秀才造反 秒針 秒針 秧針 秧針 +穆稜 穆稜 穿針 穿針 穿針引線 穿針引線 穿針走線 穿針走線 @@ -796,6 +798,7 @@ 絕才 絕才 絞包針 絞包針 給個棒錘當針認 給個棒錘當針認 +綏稜 綏稜 經世之才 經世之才 經國之才 經國之才 經濟之才 經濟之才 diff --git a/data/dictionary/STCharacters.txt b/data/dictionary/STCharacters.txt index cf7a11d0d..f68e9262c 100644 --- a/data/dictionary/STCharacters.txt +++ b/data/dictionary/STCharacters.txt @@ -254,6 +254,7 @@ 兰 蘭 关 關 兴 興 +具 具 俱 兹 茲 养 養 兽 獸 @@ -264,6 +265,7 @@ 写 寫 军 軍 农 農 +冢 冢 塚 冬 冬 鼕 冯 馮 冲 衝 沖 @@ -382,10 +384,12 @@ 吓 嚇 吕 呂 吗 嗎 +吣 吣 唚 吨 噸 听 聽 启 啓 吴 吳 +呆 呆 待 呐 吶 呒 嘸 呓 囈 @@ -458,6 +462,7 @@ 圹 壙 场 場 坏 壞 +坐 坐 座 块 塊 坚 堅 坛 壇 罈 @@ -484,6 +489,7 @@ 埚 堝 堑 塹 堕 墮 +堤 堤 隄 塆 壪 墙 牆 壮 壯 @@ -617,8 +623,8 @@ 帻 幘 帼 幗 幂 冪 -干 幹 乾 干 -并 並 併 +干 幹 乾 干 榦 +并 並 併 幷 幸 幸 倖 广 廣 广 庄 莊 @@ -715,7 +721,7 @@ 戏 戲 戗 戧 战 戰 -戚 戚 慼 +戚 戚 慼 鏚 戬 戩 戯 戱 户 戶 @@ -750,6 +756,7 @@ 拧 擰 拨 撥 择 擇 +拿 拿 拏 挂 掛 挂 挚 摯 挛 攣 @@ -766,6 +773,7 @@ 挦 撏 挨 挨 捱 挽 挽 輓 +捆 捆 綑 捝 挩 捞 撈 损 損 @@ -812,7 +820,9 @@ 斋 齋 斓 斕 斗 鬥 斗 +斤 斤 觔 斩 斬 +斫 斫 斲 断 斷 旋 旋 鏇 无 無 @@ -856,13 +866,14 @@ 板 板 闆 极 極 极 构 構 +果 果 菓 枞 樅 枢 樞 枣 棗 枥 櫪 枧 梘 枨 棖 -枪 槍 +枪 槍 鎗 枫 楓 枭 梟 柜 櫃 柜 @@ -903,6 +914,7 @@ 检 檢 棁 梲 棂 欞 +棱 棱 稜 椁 槨 椝 槼 椟 櫝 @@ -930,6 +942,7 @@ 橱 櫥 橹 櫓 橼 櫞 +檗 檗 蘗 檩 檁 欢 歡 欤 歟 @@ -956,7 +969,7 @@ 氢 氫 氩 氬 氲 氳 -汇 匯 彙 +汇 匯 彙 滙 汉 漢 汤 湯 汹 洶 @@ -1005,6 +1018,7 @@ 浓 濃 浔 潯 浕 濜 +浚 浚 濬 涂 塗 涂 涌 湧 涌 涚 涗 @@ -1040,6 +1054,7 @@ 溅 濺 溆 漵 溇 漊 +溪 溪 谿 滗 潷 滚 滾 滞 滯 @@ -1060,6 +1075,7 @@ 潍 濰 潜 潛 潴 瀦 +澄 澄 澂 澛 瀂 澜 瀾 濑 瀨 @@ -1094,8 +1110,9 @@ 焕 煥 焖 燜 焘 燾 +焰 焰 燄 煴 熅 -熏 燻 熏 +熏 燻 熏 薰 爱 愛 爷 爺 牍 牘 @@ -1117,6 +1134,7 @@ 狰 猙 狱 獄 狲 猻 +狸 狸 貍 猃 獫 猎 獵 猕 獼 @@ -1140,6 +1158,7 @@ 珑 瓏 珰 璫 珲 琿 +琅 琅 瑯 琎 璡 琏 璉 琐 瑣 @@ -1200,6 +1219,7 @@ 眍 瞘 眦 眥 眬 矓 +着 着 著 睁 睜 睐 睞 睑 瞼 @@ -1260,6 +1280,7 @@ 稣 穌 稳 穩 穑 穡 +穗 穗 繐 穞 穭 穷 窮 窃 竊 @@ -1285,6 +1306,7 @@ 筛 篩 筜 簹 筝 箏 +筱 筱 篠 筹 籌 筼 篔 签 籤 簽 @@ -1315,6 +1337,7 @@ 粽 糉 糁 糝 糇 餱 +糊 糊 餬 糍 餈 系 系 係 繫 紧 緊 @@ -1485,6 +1508,7 @@ 翘 翹 翙 翽 翚 翬 +耇 耇 耈 耢 耮 耧 耬 耸 聳 @@ -1633,11 +1657,13 @@ 蔹 蘞 蔺 藺 蔼 藹 +蔽 蔽 庇 蕰 薀 蕲 蘄 蕴 蘊 薮 藪 藓 蘚 +藤 藤 籐 藴 蘊 蘖 櫱 虏 虜 @@ -1876,6 +1902,8 @@ 谵 譫 谶 讖 谷 谷 穀 +豆 豆 荳 +象 象 像 豮 豶 贝 貝 贞 貞 @@ -2076,6 +2104,7 @@ 酂 酇 酝 醞 酦 醱 +酰 酰 醯 酱 醬 酸 酸 痠 酽 釅 @@ -2215,7 +2244,7 @@ 锁 鎖 锂 鋰 锃 鋥 -锄 鋤 +锄 鋤 耡 锅 鍋 锆 鋯 锇 鋨 @@ -2416,7 +2445,7 @@ 靓 靚 靔 靝 静 靜 -面 面 麪 +面 面 麪 麵 靥 靨 鞑 韃 鞒 鞽 @@ -2508,7 +2537,7 @@ 饫 飫 饬 飭 饭 飯 -饮 飲 +饮 飲 飮 饯 餞 饰 飾 饱 飽 @@ -2673,7 +2702,7 @@ 鲳 鯧 鲴 鯝 鲵 鯢 -鲶 鯰 +鲶 鯰 鮎 鲷 鯛 鲸 鯨 鲹 鰺 diff --git a/data/dictionary/TSCharacters.txt b/data/dictionary/TSCharacters.txt index 043da14ca..466960083 100644 --- a/data/dictionary/TSCharacters.txt +++ b/data/dictionary/TSCharacters.txt @@ -362,6 +362,7 @@ 䶲 䶲 𫜳 丟 丢 並 并 +么 么 幺 乾 干 乾 亂 乱 亙 亘 @@ -382,6 +383,7 @@ 俠 侠 俥 伡 俬 私 +俱 俱 具 倀 伥 倆 俩 倈 俫 @@ -412,6 +414,7 @@ 僂 偻 僅 仅 僉 佥 +像 像 象 僑 侨 僕 仆 僞 伪 @@ -1307,6 +1310,7 @@ 汎 泛 汙 污 決 决 +沈 沈 沉 沒 没 沖 冲 況 况 @@ -2247,6 +2251,7 @@ 萵 莴 葉 叶 葒 荭 +著 著 着 # @tofu-risk: not covered by PingFang/Source Han baseline: 𫈎 U+2B20E 葝 葝 𫈎 葤 荮 diff --git a/data/dictionary/TWVariantsRevPhrases.txt b/data/dictionary/TWVariantsRevPhrases.txt index bd85fdcbf..7667f230f 100644 --- a/data/dictionary/TWVariantsRevPhrases.txt +++ b/data/dictionary/TWVariantsRevPhrases.txt @@ -39,6 +39,7 @@ 丟巧針 丟巧針 中才 中才 中核 中核 +丹稜 丹稜 之核 之核 九針 九針 乾奴才 乾奴才 @@ -262,6 +263,7 @@ 吃吃 喫喫 吃吃 吃子 喫子 吃子 合著 合著 +合著者 合著者 同參 同參 名著 名著 向風針 向風針 @@ -360,6 +362,7 @@ 彈針 彈針 彗核 彗核 形名參同 形名參同 +待著 待著 得失參半 得失參半 微核 微核 徵才 徵才 @@ -372,6 +375,7 @@ 恃才傲物 恃才傲物 恃才矜己 恃才矜己 恃才自專 恃才自專 +惡名昭著 惡名昭著 意廣才疏 意廣才疏 愛才 愛才 愛才好士 愛才好士 @@ -728,6 +732,7 @@ 秀才造反 秀才造反 秒針 秒針 秧針 秧針 +穆稜 穆稜 積極參加 積極參加 積極參與 積極參與 穿針 穿針 @@ -746,6 +751,7 @@ 絕才 絕才 絞包針 絞包針 給個棒錘當針認 給個棒錘當針認 +綏稜 綏稜 經世之才 經世之才 經國之才 經國之才 經濟之才 經濟之才 @@ -804,6 +810,7 @@ 著書 著書 著白 著白 著稱 著稱 +著稱於世 著稱於世 著者 著者 著述 著述 著錄 著錄 @@ -819,6 +826,7 @@ 行短才高 行短才高 補血針 補血針 補針 補針 +製麵 製麵 西洋參 西洋參 見縫插針 見縫插針 討針線 討針線 @@ -986,6 +994,7 @@ 預防針 預防針 領帶針 領帶針 顯著 顯著 +顯著標志 顯著標志 風向針 風向針 風流才子 風流才子 飛針走線 飛針走線 @@ -1002,6 +1011,7 @@ 鬼才 鬼才 鬼針草 鬼針草 魚頭參政 魚頭參政 +鴻篇鉅著 鴻篇鉅著 鹽水針 鹽水針 麴秀才 麴秀才 麻醉針 麻醉針