diff --git a/README.md b/README.md index 35ef3ccf9..daf8c095d 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,9 @@ Rules: * `s2tw.json` **Simplified Chinese** to **Traditional Chinese (Taiwan Standard)** / **簡體** 到 **台灣正體** * `tw2s.json` **Traditional Chinese (Taiwan Standard)** to **Simplified Chinese** / **台灣正體** 到 **簡體** * `s2hk.json` **Simplified Chinese** to **Traditional Chinese (Hong Kong variant)** / **簡體** 到 **香港繁體** +* `s2hkp.json` **Simplified Chinese** to **Traditional Chinese (Hong Kong variant, with Hong Kong Phrases)** / **簡體** 到 **香港繁體（香港常用詞彙）** * `hk2s.json` **Traditional Chinese (Hong Kong variant)** to **Simplified Chinese** / **香港繁體** 到 **簡體** +* `hk2sp.json` **Traditional Chinese (Hong Kong variant)** to **Simplified Chinese (Mainland China Phrases)** / **香港繁體** 到 **簡體（含中國大陸常用詞彙）** * `s2twp.json` **Simplified Chinese** to **Traditional Chinese (Taiwan Standard, with Taiwan Phrases)** / **簡體** 到 **台灣正體（含台灣常用詞彙）** * `tw2sp.json` **Traditional Chinese (Taiwan Standard)** to **Simplified Chinese (Mainland China Phrases)** / **台灣正體** 到 **簡體（含中國大陸常用詞彙）** * `t2tw.json` **Traditional Chinese (OpenCC Standard)** to **Traditional Chinese (Taiwan Standard)** / **OpenCC 標準繁體** 到 **台灣正體** diff --git a/data/CMakeLists.txt b/data/CMakeLists.txt index a1dc057b7..ec8a178a6 100644 --- a/data/CMakeLists.txt +++ b/data/CMakeLists.txt @@ -20,6 +20,8 @@ set( HKVariantsPhrases HKVariants HKVariantsRevPhrases + HKPhrases + HKPhrasesRev JPVariants JPShinjitaiCharacters JPShinjitaiPhrases @@ -157,9 +159,11 @@ endforeach(DICT) set(CONFIG_FILES config/hk2s.json + config/hk2sp.json config/hk2t.json config/jp2t.json config/s2hk.json + config/s2hkp.json config/s2t.json config/s2tw.json config/s2twp.json diff --git a/data/config/hk2sp.json b/data/config/hk2sp.json new file mode 100644 index 000000000..d41c6e252 --- /dev/null +++ b/data/config/hk2sp.json @@ -0,0 +1,33 @@ +{ + "name": "Traditional Chinese (Hong Kong variant) to Simplified Chinese (with phrases)", + "segmentation": { + "type": "mmseg", + "dict": { "type": "ocd2", "file": "TSPhrases.ocd2" } + }, + "conversion_chain": [ + { + "dict": { + "type": "group", + "dicts": [ + { "type": "ocd2", "file": "HKPhrasesRev.ocd2" }, + { "type": "ocd2", "file": "HKVariantsRevPhrases.ocd2" }, + { "type": "ocd2", "file": "HKVariantsRev.ocd2" } + ] + } + }, + { + "dict": { + "type": "group", + "dicts": [ + { "type": "ocd2", "file": "TSPhrases.ocd2" }, + { + "type": "ocd2", + "file": "TSCharactersExt.ocd2", + "may_output_tofu": true + }, + { "type": "ocd2", "file": "TSCharacters.ocd2" } + ] + } + } + ] +} diff --git a/data/config/s2hkp.json b/data/config/s2hkp.json new file mode 100644 index 000000000..d4f4917be --- /dev/null +++ b/data/config/s2hkp.json @@ -0,0 +1,28 @@ +{ + "name": "Simplified Chinese to Traditional Chinese (Hong Kong variant, with phrases)", + "segmentation": { + "type": "mmseg", + "dict": { "type": "ocd2", "file": "STPhrases.ocd2" } + }, + "conversion_chain": [ + { + "dict": { + "type": "group", + "dicts": [ + { "type": "ocd2", "file": "STPhrases.ocd2" }, + { "type": "ocd2", "file": "STCharacters.ocd2" } + ] + } + }, + { + "dict": { + "type": "group", + "dicts": [ + { "type": "ocd2", "file": "HKPhrases.ocd2" }, + { "type": "ocd2", "file": "HKVariantsPhrases.ocd2" }, + { "type": "ocd2", "file": "HKVariants.ocd2" } + ] + } + } + ] +} diff --git a/data/dictionary/BUILD.bazel b/data/dictionary/BUILD.bazel index 7f7305351..0dc995be5 100644 --- a/data/dictionary/BUILD.bazel +++ b/data/dictionary/BUILD.bazel @@ -94,10 +94,12 @@ filegroup( ] cc_test( - name = "dictionary_TWPhrases_reverse_mapping_test", + name = "dictionary_phrases_reverse_mapping_test", size = "small", srcs = ["DictionaryReverseMappingTest.cpp"], data = [ + "HKPhrases.txt", + "HKPhrasesRev.txt", "TWPhrases.txt", "TWPhrasesRev.txt", ], @@ -110,10 +112,11 @@ cc_test( ) cc_test( - name = "dictionary_TWPhrases_segmentation_test", + name = "dictionary_phrases_segmentation_test", size = "small", - srcs = ["DictionaryTaiwanPhraseSegmentationTest.cpp"], + srcs = ["DictionaryPhraseSegmentationTest.cpp"], data = [ + "HKPhrases.txt", "STPhrases.txt", "TWPhrases.txt", ], @@ -192,8 +195,8 @@ test_suite( ":dictionary_" + txt[:-4] + "_test" for txt in TEXT_DICTS ] + [ - ":dictionary_TWPhrases_reverse_mapping_test", - ":dictionary_TWPhrases_segmentation_test", + ":dictionary_phrases_reverse_mapping_test", + ":dictionary_phrases_segmentation_test", ":dictionary_compatibility_ideographs_test", ":dictionary_phrase_character_dependency_test", ":dictionary_variant_rev_phrases_test", diff --git a/data/dictionary/DictionaryTaiwanPhraseSegmentationTest.cpp b/data/dictionary/DictionaryPhraseSegmentationTest.cpp similarity index 84% rename from data/dictionary/DictionaryTaiwanPhraseSegmentationTest.cpp rename to data/dictionary/DictionaryPhraseSegmentationTest.cpp index 78d09caf3..ca4cf3415 100644 --- a/data/dictionary/DictionaryTaiwanPhraseSegmentationTest.cpp +++ b/data/dictionary/DictionaryPhraseSegmentationTest.cpp @@ -143,27 +143,26 @@ bool FindSTPhraseValueFragment(const std::string& key, return false; } -TEST(DictionaryTaiwanPhraseSegmentationTest, - TWPhrasesKeysAreCoveredBySTPhraseValues) { +void ExpectProperSegmentationCoverage(const std::string& phrasesName) { std::unique_ptr runfiles(Runfiles::CreateForTest()); ASSERT_NE(nullptr, runfiles); const std::string stPhrasesFile = runfiles->Rlocation("_main/data/dictionary/STPhrases.txt"); - const std::string twPhrasesFile = - runfiles->Rlocation("_main/data/dictionary/TWPhrases.txt"); + const std::string phrasesFile = + runfiles->Rlocation("_main/data/dictionary/" + phrasesName + ".txt"); try { - LexiconPtr twPhrases = LoadLexicon(twPhrasesFile); - ASSERT_NE(twPhrases, nullptr); + LexiconPtr phrases = LoadLexicon(phrasesFile); + ASSERT_NE(phrases, nullptr); std::vector stValues = LoadSTPhraseValues(stPhrasesFile); ASSERT_FALSE(stValues.empty()); std::ostringstream missing; size_t missingCount = 0; - for (size_t i = 0; i < twPhrases->Length(); ++i) { - const std::string key = twPhrases->At(i)->Key(); + for (size_t i = 0; i < phrases->Length(); ++i) { + const std::string key = phrases->At(i)->Key(); if (IsCoveredBySTPhraseValue(key, stValues)) { continue; } @@ -174,27 +173,37 @@ TEST(DictionaryTaiwanPhraseSegmentationTest, } ++missingCount; - missing << "TWPhrases key \"" << key + missing << phrasesName << " key \"" << key << "\" is not covered by any STPhrases value.\n" << " Conflicting STPhrases record: STPhrases.txt:" << risk.lineNumber << " \"" << risk.key << "\" -> \"" << risk.value << "\"\n" << " The existing value appears as a " << risk.category - << " fragment of the Taiwan phrase key.\n"; + << " fragment of the " << phrasesName << " key.\n"; } EXPECT_EQ(0U, missingCount) - << "Potential missing STPhrases entries for s2twp segmentation:\n" + << "Potential missing STPhrases entries for " << phrasesName << ":\n" << missing.str(); } catch (const Exception& ex) { FAIL() << "Exception: " << ex.what(); } catch (const std::exception& ex) { FAIL() << "std::exception: " << ex.what(); } catch (...) { - FAIL() << "Unknown exception thrown during Taiwan phrase segmentation " + FAIL() << "Unknown exception thrown during " << phrasesName << " segmentation " "check."; } } +TEST(DictionaryPhraseSegmentationTest, + HKPhrasesKeysAreCoveredBySTPhraseValues) { + ExpectProperSegmentationCoverage("HKPhrases"); +} + +TEST(DictionaryPhraseSegmentationTest, + TWPhrasesKeysAreCoveredBySTPhraseValues) { + ExpectProperSegmentationCoverage("TWPhrases"); +} + } // namespace } // namespace opencc diff --git a/data/dictionary/DictionaryReverseMappingTest.cpp b/data/dictionary/DictionaryReverseMappingTest.cpp index 7d5d45cde..4b612a8c6 100644 --- a/data/dictionary/DictionaryReverseMappingTest.cpp +++ b/data/dictionary/DictionaryReverseMappingTest.cpp @@ -27,14 +27,14 @@ static FILE* OpenFile(const std::string& path) { #endif } -TEST(DictionaryReverseMappingTest, TWPhrasesReverseMapping) { +void ExpectRevComplete(const std::string& phrasesName) { std::unique_ptr runfiles(Runfiles::CreateForTest()); ASSERT_NE(nullptr, runfiles); - const std::string twPhrasesFile = - runfiles->Rlocation("_main/data/dictionary/TWPhrases.txt"); - const std::string twPhrasesRevFile = - runfiles->Rlocation("_main/data/dictionary/TWPhrasesRev.txt"); + const std::string phrasesFile = + runfiles->Rlocation("_main/data/dictionary/" + phrasesName + ".txt"); + const std::string phrasesRevFile = + runfiles->Rlocation("_main/data/dictionary/" + phrasesName + "Rev" + ".txt"); auto loadLexicon = [](const std::string& path) -> LexiconPtr { FILE* fp = OpenFile(path); @@ -62,28 +62,28 @@ TEST(DictionaryReverseMappingTest, TWPhrasesReverseMapping) { }; try { - LexiconPtr twPhrases = loadLexicon(twPhrasesFile); - LexiconPtr twPhrasesRev = loadLexicon(twPhrasesRevFile); + LexiconPtr twPhrases = loadLexicon(phrasesFile); + LexiconPtr twPhrasesRev = loadLexicon(phrasesRevFile); ASSERT_NE(twPhrases, nullptr); ASSERT_NE(twPhrasesRev, nullptr); - auto twMap = buildMap(twPhrases); - auto twRevMap = buildMap(twPhrasesRev); + auto phrasesMap = buildMap(twPhrases); + auto phrasesRevMap = buildMap(twPhrasesRev); - for (const auto& entry : twMap) { + for (const auto& entry : phrasesMap) { const std::string& key = entry.first; for (const auto& value : entry.second) { - auto it = twRevMap.find(value); - EXPECT_TRUE(it != twRevMap.end() && it->second.count(key) > 0) + auto it = phrasesRevMap.find(value); + EXPECT_TRUE(it != phrasesRevMap.end() && it->second.count(key) > 0) << "Missing reverse mapping: " << key << " -> " << value; } } - for (const auto& entry : twRevMap) { + for (const auto& entry : phrasesRevMap) { const std::string& key = entry.first; for (const auto& value : entry.second) { - auto it = twMap.find(value); - EXPECT_TRUE(it != twMap.end() && it->second.count(key) > 0) + auto it = phrasesMap.find(value); + EXPECT_TRUE(it != phrasesMap.end() && it->second.count(key) > 0) << "Missing reverse mapping: " << key << " -> " << value; } } @@ -96,5 +96,13 @@ TEST(DictionaryReverseMappingTest, TWPhrasesReverseMapping) { } } +TEST(DictionaryReverseMappingTest, HKPhrasesReverseMapping) { + ExpectRevComplete("HKPhrases"); +} + +TEST(DictionaryReverseMappingTest, TWPhrasesReverseMapping) { + ExpectRevComplete("TWPhrases"); +} + } // namespace } // namespace opencc diff --git a/data/dictionary/HKPhrases.txt b/data/dictionary/HKPhrases.txt new file mode 100644 index 000000000..a9fba934f --- /dev/null +++ b/data/dictionary/HKPhrases.txt @@ -0,0 +1,16 @@ +# Open Chinese Convert (OpenCC) Dictionary +# File: HKPhrases.txt +# Format: key value(s) (values separated by spaces) +# License: Apache-2.0 (see LICENSE) +# Source: https://github.com/ByVoid/OpenCC +# Used in configs: s2hkp.json (via HKPhrases.ocd2) + +寬帶 寬頻 +幾率 機率 概率 +搜索 搜尋 +操作系統 作業系統 +文件夾 資料夾 +服務器 伺服器 +概率 概率 機會率 +編程語言 程式語言 +隱私權 私隱權 diff --git a/data/dictionary/HKPhrasesRev.txt b/data/dictionary/HKPhrasesRev.txt new file mode 100644 index 000000000..1978fcc5e --- /dev/null +++ b/data/dictionary/HKPhrasesRev.txt @@ -0,0 +1,17 @@ +# Open Chinese Convert (OpenCC) Dictionary +# File: HKPhrasesRev.txt +# Format: key value(s) (values separated by spaces) +# License: Apache-2.0 (see LICENSE) +# Source: https://github.com/ByVoid/OpenCC +# Used in configs: hk2sp.json (via HKPhrasesRev.ocd2) + +伺服器 服務器 +作業系統 操作系統 +寬頻 寬帶 +搜尋 搜索 +概率 概率 幾率 +機會率 概率 +機率 幾率 +私隱權 隱私權 +程式語言 編程語言 +資料夾 文件夾 diff --git a/node/configs.gypi b/node/configs.gypi index 796107621..b36b69d4f 100644 --- a/node/configs.gypi +++ b/node/configs.gypi @@ -6,9 +6,11 @@ "destination": "<(PRODUCT_DIR)", "files": [ "../data/config/hk2s.json", + "../data/config/hk2sp.json", "../data/config/hk2t.json", "../data/config/jp2t.json", "../data/config/s2hk.json", + "../data/config/s2hkp.json", "../data/config/s2t.json", "../data/config/s2tw.json", "../data/config/s2twp.json", diff --git a/node/dicts.gypi b/node/dicts.gypi index 8d62d48ae..b173807a0 100644 --- a/node/dicts.gypi +++ b/node/dicts.gypi @@ -156,6 +156,14 @@ "inputs": ["<(input)"], "outputs": ["<(output_prefix)HKVariantsRevPhrases.ocd2"], "action": ["node", "<(cmd)", "<(input)", "<@(_outputs)"] + }, { + "action_name": "HKPhrases", + "variables": { + "input": "<(input_prefix)HKPhrases.txt", + }, + "inputs": ["<(input)"], + "outputs": ["<(output_prefix)HKPhrases.ocd2"], + "action": ["node", "<(cmd)", "<(input)", "<@(_outputs)"] }, { "action_name": "HKVariantsRev.txt", "variables": { @@ -172,6 +180,14 @@ "inputs": ["<(input)"], "outputs": ["<(output_prefix)HKVariantsRev.ocd2"], "action": ["node", "<(cmd)", "<(input)", "<@(_outputs)"] + }, { + "action_name": "HKPhrasesRev", + "variables": { + "input": "<(input_prefix)HKPhrasesRev.txt", + }, + "inputs": ["<(input)"], + "outputs": ["<(output_prefix)HKPhrasesRev.ocd2"], + "action": ["node", "<(cmd)", "<(input)", "<@(_outputs)"] }, { "action_name": "JPVariantsRev.txt", "variables": { diff --git a/plugins/jieba/CMakeLists.txt b/plugins/jieba/CMakeLists.txt index 3c0248961..8fd69c653 100644 --- a/plugins/jieba/CMakeLists.txt +++ b/plugins/jieba/CMakeLists.txt @@ -251,7 +251,9 @@ if (BUILD_OPENCC_JIEBA_PLUGIN AND OPENCC_ENABLE_INSTALL) install( FILES + ${CMAKE_CURRENT_SOURCE_DIR}/data/config/hk2sp_jieba.json ${CMAKE_CURRENT_SOURCE_DIR}/data/config/s2hk_jieba.json + ${CMAKE_CURRENT_SOURCE_DIR}/data/config/s2hkp_jieba.json ${CMAKE_CURRENT_SOURCE_DIR}/data/config/s2t_jieba.json ${CMAKE_CURRENT_SOURCE_DIR}/data/config/s2tw_jieba.json ${CMAKE_CURRENT_SOURCE_DIR}/data/config/s2twp_jieba.json diff --git a/plugins/jieba/data/config/hk2sp_jieba.json b/plugins/jieba/data/config/hk2sp_jieba.json new file mode 100644 index 000000000..a8cf495db --- /dev/null +++ b/plugins/jieba/data/config/hk2sp_jieba.json @@ -0,0 +1,36 @@ +{ + "name": "Traditional Chinese (Hong Kong variant) to Simplified Chinese (with phrases, Jieba Segmentation - Experimental)", + "segmentation": { + "type": "jieba", + "resources": { + "dict_path": "jieba_dict/jieba_merged.ocd2", + "model_path": "jieba_dict/hmm_model.utf8" + } + }, + "conversion_chain": [ + { + "dict": { + "type": "group", + "dicts": [ + { "type": "ocd2", "file": "HKPhrasesRev.ocd2" }, + { "type": "ocd2", "file": "HKVariantsRevPhrases.ocd2" }, + { "type": "ocd2", "file": "HKVariantsRev.ocd2" } + ] + } + }, + { + "dict": { + "type": "group", + "dicts": [ + { "type": "ocd2", "file": "TSPhrases.ocd2" }, + { + "type": "ocd2", + "file": "TSCharactersExt.ocd2", + "may_output_tofu": true + }, + { "type": "ocd2", "file": "TSCharacters.ocd2" } + ] + } + } + ] +} diff --git a/plugins/jieba/data/config/s2hkp_jieba.json b/plugins/jieba/data/config/s2hkp_jieba.json new file mode 100644 index 000000000..abac9ddb7 --- /dev/null +++ b/plugins/jieba/data/config/s2hkp_jieba.json @@ -0,0 +1,31 @@ +{ + "name": "Simplified Chinese to Traditional Chinese (Hong Kong variant, with phrases, Jieba Segmentation - Experimental)", + "segmentation": { + "type": "jieba", + "resources": { + "dict_path": "jieba_dict/jieba_merged.ocd2", + "model_path": "jieba_dict/hmm_model.utf8" + } + }, + "conversion_chain": [ + { + "dict": { + "type": "group", + "dicts": [ + { "type": "ocd2", "file": "STPhrases.ocd2" }, + { "type": "ocd2", "file": "STCharacters.ocd2" } + ] + } + }, + { + "dict": { + "type": "group", + "dicts": [ + { "type": "ocd2", "file": "HKPhrases.ocd2" }, + { "type": "ocd2", "file": "HKVariantsPhrases.ocd2" }, + { "type": "ocd2", "file": "HKVariants.ocd2" } + ] + } + } + ] +} diff --git a/plugins/jieba/node/build.js b/plugins/jieba/node/build.js index 030a87630..3659bb04c 100644 --- a/plugins/jieba/node/build.js +++ b/plugins/jieba/node/build.js @@ -127,7 +127,9 @@ console.log(`Copied ${libName} to prebuilds/${target}/`); const configSrcDir = path.join(rootDir, 'plugins/jieba/data/config'); const configs = [ + 'hk2sp_jieba.json', 's2hk_jieba.json', + 's2hkp_jieba.json', 's2t_jieba.json', 's2tw_jieba.json', 's2twp_jieba.json', diff --git a/plugins/jieba/tests/data/jieba_comparison_testcases.json b/plugins/jieba/tests/data/jieba_comparison_testcases.json index da8b10ee2..de9ffc0cc 100644 --- a/plugins/jieba/tests/data/jieba_comparison_testcases.json +++ b/plugins/jieba/tests/data/jieba_comparison_testcases.json @@ -27,7 +27,8 @@ "input": "㑮", "expected_segmentation": "㑮", "expected": { - "tw2sp_jieba": "㑮" + "tw2sp_jieba": "㑮", + "hk2sp_jieba": "㑮" } }, { diff --git a/test/testcases/testcases.json b/test/testcases/testcases.json index 0989db8ac..2670a19bc 100755 --- a/test/testcases/testcases.json +++ b/test/testcases/testcases.json @@ -1154,6 +1154,26 @@ "tw2s": "胺基酸 丙胺酸 精胺酸 天冬醯胺 天门冬醯胺 天冬胺酸 天门冬胺酸 胱胺酸 麸胺酸 麸醯胺酸 甘胺酸 组胺酸 异白胺酸 白胺酸 离胺酸 甲硫胺酸 苯丙胺酸 脯胺酸 丝胺酸 苏胺酸 色胺酸 酪胺酸 缬胺酸", "tw2sp": "氨基酸 丙氨酸 精氨酸 天冬酰胺 天门冬酰胺 天冬氨酸 天门冬氨酸 胱氨酸 谷氨酸 谷氨酰胺 甘氨酸 组氨酸 异亮氨酸 亮氨酸 赖氨酸 甲硫氨酸 苯丙氨酸 脯氨酸 丝氨酸 苏氨酸 色氨酸 酪氨酸 缬氨酸" } + }, + { + "id": "BYVoid_OpenCC_Issue_506", + "input": "几率 機率 機會率 概率", + "expected": { + "s2t": "幾率 機率 機會率 概率", + "s2tw": "幾率 機率 機會率 概率", + "s2twp": "機率 機率 機會率 機率", + "s2hk": "幾率 機率 機會率 概率", + "s2hkp": "機率 機率 機會率 概率", + "t2s": "几率 机率 机会率 概率", + "t2tw": "几率 機率 機會率 概率", + "t2hk": "几率 機率 機會率 概率", + "tw2s": "几率 机率 机会率 概率", + "tw2sp": "几率 几率 机会率 概率", + "tw2t": "几率 機率 機會率 概率", + "hk2s": "几率 机率 机会率 概率", + "hk2sp": "几率 几率 概率 概率", + "hk2t": "几率 機率 機會率 概率" + } } ] }