diff --git a/src/BUILD.bazel b/src/BUILD.bazel index d38d34453..823cedd88 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -40,6 +40,7 @@ cc_library( ":common_lib", ":config_lib", ":conversion_lib", + ":conversion_candidates_lib", ":conversion_chain_lib", ":converter_lib", ":darts_dict_lib", @@ -126,15 +127,24 @@ diff_test( file2 = "opencc_config_schema.inc", ) +# Private header shared between Config.cpp (which builds the converter) and +# conversion_candidates_lib (which unwraps the normalization pre-pass). cc_library( - name = "config_lib", - srcs = [ - "Config.cpp", - "ConfigBasedConverter.hpp", + name = "config_based_converter_lib", + hdrs = ["ConfigBasedConverter.hpp"], + visibility = ["//src:__pkg__"], + deps = [ + ":converter_lib", ], +) + +cc_library( + name = "config_lib", + srcs = ["Config.cpp"], hdrs = ["Config.hpp"], deps = [ ":common_lib", + ":config_based_converter_lib", ":config_schema_inc_lib", ":conversion_chain_lib", ":converter_lib", @@ -275,6 +285,47 @@ cc_library( ], ) +# Internal, unstable candidate-enumeration API (GetAllConversions). The header +# is not installed by the CMake build and the function is not part of the +# OPENCC_ABI_VERSION contract; visibility is restricted to this package so no +# other Bazel target can depend on it directly until the API is promoted. +cc_library( + name = "conversion_candidates_lib", + srcs = ["ConversionCandidates.cpp"], + hdrs = ["ConversionCandidates.hpp"], + visibility = ["//src:__pkg__"], + deps = [ + ":common_lib", + ":config_based_converter_lib", + ":conversion_lib", + ":conversion_chain_lib", + ":converter_lib", + ":dict_lib", + ":dict_entry_lib", + ], +) + +cc_test( + name = "conversion_candidates_test", + size = "small", + srcs = ["ConversionCandidatesTest.cpp"], + deps = [ + ":config_based_converter_lib", + ":conversion_candidates_lib", + ":conversion_lib", + ":conversion_chain_lib", + ":converter_lib", + ":lexicon_lib", + ":max_match_segmentation_lib", + ":pipeline_converter_lib", + ":single_stage_converter_lib", + ":test_utils_lib", + ":test_utils_utf8_lib", + ":text_dict_lib", + "@googletest//:gtest_main", + ], +) + cc_library( name = "single_stage_converter_lib", srcs = ["SingleStageConverter.cpp"], diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 17538302d..ec6361936 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -45,6 +45,7 @@ set( set( LIBOPENCC_PRIVATE_HEADERS ConfigBasedConverter.hpp + ConversionCandidates.hpp DictConverter.hpp PhraseExtract.hpp PipelineConverter.hpp @@ -59,6 +60,7 @@ set( LIBOPENCC_SOURCES Config.cpp Conversion.cpp + ConversionCandidates.cpp ConversionChain.cpp Converter.cpp Dict.cpp @@ -85,6 +87,7 @@ set( set(UNITTESTS ConfigTest + ConversionCandidatesTest ConversionChainTest ConversionInspectionTest ConversionTest diff --git a/src/ConfigBasedConverter.hpp b/src/ConfigBasedConverter.hpp index c8aa6492e..0cb9ee579 100644 --- a/src/ConfigBasedConverter.hpp +++ b/src/ConfigBasedConverter.hpp @@ -63,6 +63,14 @@ class ConfigBasedConverter : public Converter { return mainConverter->GetConversionChain(); } + /** + * Returns the normalization pre-pass converter. Used by GetAllConversions() + * to normalize a word before walking the main chain, mirroring Convert(). + */ + const ConverterPtr& GetNormalizationConverter() const { + return normConverter; + } + private: const ConverterPtr normConverter; const ConverterPtr mainConverter; diff --git a/src/ConversionCandidates.cpp b/src/ConversionCandidates.cpp new file mode 100644 index 000000000..63bd2c986 --- /dev/null +++ b/src/ConversionCandidates.cpp @@ -0,0 +1,100 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2026 Carbo Kuo and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ConversionCandidates.hpp" + +#include + +#include "Common.hpp" +#include "ConfigBasedConverter.hpp" +#include "Conversion.hpp" +#include "ConversionChain.hpp" +#include "Converter.hpp" +#include "Dict.hpp" +#include "DictEntry.hpp" +#include "Optional.hpp" + +namespace opencc { + +std::vector GetAllConversions(const Converter& converter, + std::string_view word) { + const ConversionChainPtr chain = converter.GetConversionChain(); + if (chain == nullptr) { + return {}; + } + + // A config-based converter runs a normalization pre-pass (e.g. CJK + // Compatibility Ideographs) before its main chain, but GetConversionChain() + // exposes only the main chain. Mirror Convert() by normalizing first, so a + // word whose dictionary key appears only after normalization is still found. + std::string normalized; + if (const auto* configBased = + dynamic_cast(&converter)) { + normalized = configBased->GetNormalizationConverter()->Convert(word); + word = normalized; + } + + std::vector currentWords{std::string(word)}; + bool matched = false; + for (const ConversionPtr& conversion : chain->GetConversions()) { + const DictPtr dict = conversion->GetDict(); + if (dict == nullptr) { + return {}; + } + std::unordered_set seen; + std::vector nextWords; + for (const std::string& currentWord : currentWords) { + const Optional item = dict->Match(currentWord); + if (item.IsNull()) { + // No exact match: convert greedily by longest prefix so a partially + // convertible word can still flow through the rest of the chain. Even + // when the result is unchanged we keep it, because a later dictionary + // may still convert it. Conversion::Convert also handles ideographic + // description sequences and truncated UTF-8 safely. + std::string buffer = conversion->Convert(currentWord); + if (seen.insert(buffer).second) { + nextWords.push_back(std::move(buffer)); + } + continue; + } + matched = true; + if (item.Get()->NumValues() == 0) { + // A key-only entry converts the word to itself, matching Convert(), + // which falls back to the key via DictEntry::GetDefault(). + if (seen.insert(currentWord).second) { + nextWords.push_back(currentWord); + } + continue; + } + for (const std::string& value : item.Get()->Values()) { + if (seen.insert(value).second) { + nextWords.push_back(value); + } + } + } + currentWords.swap(nextWords); + } + + if (!matched) { + // No dictionary in the chain contains the word. + return {}; + } + return currentWords; +} + +} // namespace opencc diff --git a/src/ConversionCandidates.hpp b/src/ConversionCandidates.hpp new file mode 100644 index 000000000..6226c1214 --- /dev/null +++ b/src/ConversionCandidates.hpp @@ -0,0 +1,73 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2026 Carbo Kuo and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "Export.hpp" + +namespace opencc { + +class Converter; + +/** + * Enumerates every candidate form of a single word produced by running it + * through @p converter's conversion chain. + * + * Unlike Converter::Convert(), which segments the whole text and yields exactly + * one output string, this walks @p word through each dictionary in the chain + * and keeps @b all branch values. For example, @c s2t expands @c 里 to both + * @c 里 and @c 裏, then @c t2tw passes @c 里 through unchanged and converts + * @c 裏 to @c 裡. This mirrors the behaviour input-method engines (e.g. + * librime's @c ConvertWord) need to offer users every plausible conversion of a + * single word. + * + * Matching a word against a dictionary: + * - On an exact match, every value of the entry becomes a candidate; a + * key-only entry (no values) yields the word itself, matching Convert(). + * - Otherwise the word is converted with Conversion::Convert (greedy + * longest-prefix, each matched prefix's default value), so a partially + * convertible word still flows through the remaining dictionaries in the + * chain. + * + * A converter loaded from a config with a @c normalization step (e.g. + * @c s2t.json) normalizes @p word first, mirroring Converter::Convert(). + * + * @param converter Source of the conversion chain. A converter without a + * single chain (e.g. @c PipelineConverter, whose GetConversionChain() + * returns @c nullptr) yields an empty result. + * @param word UTF-8 text of a single word; need not be null-terminated. + * @return Candidate forms in discovery order with duplicates removed. Empty + * when no dictionary in the chain contains @p word, matching librime's + * convention of reporting "not found". + * + * @note Internal, unstable API with no compatibility guarantee. This header + * is not installed by CMake and the function is not part of the + * OPENCC_ABI_VERSION contract; like other non-installed headers + * (e.g. PhraseExtract.hpp) the symbol is still OPENCC_EXPORT so unit tests + * can link against a shared libopencc on Windows, which means the symbol is + * technically reachable by forward declaration — binding to it outside this + * repository is unsupported and may break without an ABI bump. + */ +OPENCC_EXPORT std::vector +GetAllConversions(const Converter& converter, std::string_view word); + +} // namespace opencc diff --git a/src/ConversionCandidatesTest.cpp b/src/ConversionCandidatesTest.cpp new file mode 100644 index 000000000..cb4a897a9 --- /dev/null +++ b/src/ConversionCandidatesTest.cpp @@ -0,0 +1,179 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2026 Carbo Kuo and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ConversionCandidates.hpp" + +#include "ConfigBasedConverter.hpp" +#include "Conversion.hpp" +#include "ConversionChain.hpp" +#include "Converter.hpp" +#include "Lexicon.hpp" +#include "MaxMatchSegmentation.hpp" +#include "PipelineConverter.hpp" +#include "SingleStageConverter.hpp" +#include "TestUtils.hpp" +#include "TestUtilsUTF8.hpp" +#include "TextDict.hpp" + +namespace opencc { + +class ConversionCandidatesTest : public ::testing::Test { +protected: + static DictPtr MakeDict( + const std::vector>>& + entries) { + LexiconPtr lexicon(new Lexicon); + for (const auto& entry : entries) { + lexicon->Add(DictEntryFactory::New(entry.first, entry.second)); + } + lexicon->Sort(); + return DictPtr(new TextDict(lexicon)); + } + + static ConverterPtr MakeConverter(const std::list& dicts) { + std::list conversions; + for (const DictPtr& dict : dicts) { + conversions.push_back(ConversionPtr(new Conversion(dict))); + } + ConversionChainPtr chain(new ConversionChain(conversions)); + // Segmentation is irrelevant to word-level candidate lookup; any dict works. + SegmentationPtr segmentation( + new MaxMatchSegmentation(dicts.empty() ? MakeDict({}) : dicts.front())); + return ConverterPtr(new SingleStageConverter(segmentation, chain)); + } +}; + +// A single dictionary with multiple values returns every value. +TEST_F(ConversionCandidatesTest, SingleDictMultipleValues) { + const ConverterPtr converter = + MakeConverter({MakeDict({{utf8("里"), {utf8("里"), utf8("裏")}}})}); + const std::vector candidates = + GetAllConversions(*converter, utf8("里")); + EXPECT_EQ((std::vector{utf8("里"), utf8("裏")}), candidates); +} + +// Candidates from an earlier dictionary flow independently through a later one: +// s2t expands 里 -> {里, 裏}; t2tw then passes 里 through and maps 裏 -> 裡. +TEST_F(ConversionCandidatesTest, ChainedExpansionCarriesBranches) { + const ConverterPtr converter = MakeConverter({ + MakeDict({{utf8("里"), {utf8("里"), utf8("裏")}}}), + MakeDict({{utf8("裏"), {utf8("裡")}}}), + }); + const std::vector candidates = + GetAllConversions(*converter, utf8("里")); + EXPECT_EQ((std::vector{utf8("里"), utf8("裡")}), candidates); +} + +// Duplicate branches collapse: both 麵 and 面 map to 面 downstream. +TEST_F(ConversionCandidatesTest, DuplicateCandidatesAreDeduped) { + const ConverterPtr converter = MakeConverter({ + MakeDict({{utf8("面"), {utf8("麵"), utf8("面")}}}), + MakeDict({{utf8("麵"), {utf8("面")}}}), + }); + const std::vector candidates = + GetAllConversions(*converter, utf8("面")); + EXPECT_EQ((std::vector{utf8("面")}), candidates); +} + +// A word that no dictionary in the chain contains yields an empty result. +TEST_F(ConversionCandidatesTest, WordNotInAnyDictReturnsEmpty) { + const ConverterPtr converter = + MakeConverter({MakeDict({{utf8("里"), {utf8("裏")}}})}); + EXPECT_TRUE(GetAllConversions(*converter, utf8("国")).empty()); +} + +// Only a prefix (not the whole word) matches: no exact match means "not found", +// matching librime's ConvertWord contract. +TEST_F(ConversionCandidatesTest, PrefixOnlyMatchReturnsEmpty) { + const ConverterPtr converter = + MakeConverter({MakeDict({{utf8("里"), {utf8("裏")}}})}); + EXPECT_TRUE(GetAllConversions(*converter, utf8("里面")).empty()); +} + +// An exact match on a longer word still carries an unconvertible tail through a +// later dictionary via longest-prefix conversion. +TEST_F(ConversionCandidatesTest, PartialConversionInLaterDict) { + const ConverterPtr converter = MakeConverter({ + MakeDict({{utf8("里面"), {utf8("裏面")}}}), + MakeDict({{utf8("裏"), {utf8("裡")}}}), + }); + const std::vector candidates = + GetAllConversions(*converter, utf8("里面")); + // Stage 1: 里面 -> 裏面. Stage 2: 裏 -> 裡 (prefix), 面 copied -> 裡面. + EXPECT_EQ((std::vector{utf8("裡面")}), candidates); +} + +// A converter without a single chain (PipelineConverter) yields no candidates. +TEST_F(ConversionCandidatesTest, PipelineConverterYieldsEmpty) { + const ConverterPtr stage = + MakeConverter({MakeDict({{utf8("里"), {utf8("裏")}}})}); + ConverterPtr pipeline(new PipelineConverter({stage})); + EXPECT_EQ(nullptr, pipeline->GetConversionChain()); + EXPECT_TRUE(GetAllConversions(*pipeline, utf8("里")).empty()); +} + +// A key-only entry (no values) converts the word to itself, matching +// Convert(), instead of silently dropping the branch. +TEST_F(ConversionCandidatesTest, KeyOnlyEntryPassesWordThrough) { + const ConverterPtr converter = + MakeConverter({MakeDict({{utf8("通道"), {}}})}); + const std::vector candidates = + GetAllConversions(*converter, utf8("通道")); + EXPECT_EQ((std::vector{utf8("通道")}), candidates); +} + +// Input ending in a truncated UTF-8 sequence must not read out of bounds; the +// longest-prefix fallback delegates to Conversion::Convert, which clamps. +TEST_F(ConversionCandidatesTest, TruncatedUtf8InputIsSafe) { + const ConverterPtr converter = + MakeConverter({MakeDict({{utf8("里"), {utf8("裏")}}})}); + // First two bytes of 里 (0xE9 0x87 0x8C): an incomplete 3-byte sequence. + EXPECT_TRUE(GetAllConversions(*converter, "\xE9\x87").empty()); + // Truncated tail after a convertible prefix: no exact match anywhere. + EXPECT_TRUE(GetAllConversions(*converter, utf8("里") + "\xE9\x87").empty()); +} + +// An ideographic description sequence flowing through a later dictionary's +// longest-prefix fallback stays atomic, matching Converter::Convert. +TEST_F(ConversionCandidatesTest, IdsKeptAtomicInLaterDict) { + const ConverterPtr converter = MakeConverter({ + MakeDict({{utf8("里字"), {utf8("⿱艹里")}}}), + MakeDict({{utf8("里"), {utf8("裏")}}}), + }); + const std::vector candidates = + GetAllConversions(*converter, utf8("里字")); + // Stage 2 must not convert the 里 inside the IDS ⿱艹里. + EXPECT_EQ((std::vector{utf8("⿱艹里")}), candidates); +} + +// A config-based converter's normalization pre-pass is applied before the main +// chain, mirroring Convert(): a CJK Compatibility Ideograph key is normalized +// first and then expanded by the main chain. +TEST_F(ConversionCandidatesTest, NormalizationPrePassApplied) { + // \xEF\xA7\xA9 is U+F9E9, the compatibility ideograph for 里. + const ConverterPtr norm = + MakeConverter({MakeDict({{"\xEF\xA7\xA9", {utf8("里")}}})}); + const ConverterPtr main = + MakeConverter({MakeDict({{utf8("里"), {utf8("里"), utf8("裏")}}})}); + const ConverterPtr converter(new ConfigBasedConverter(norm, main)); + const std::vector candidates = + GetAllConversions(*converter, "\xEF\xA7\xA9"); + EXPECT_EQ((std::vector{utf8("里"), utf8("裏")}), candidates); +} + +} // namespace opencc