diff --git a/README.md b/README.md index f9086fc26..8cfd254b1 100644 --- a/README.md +++ b/README.md @@ -260,13 +260,21 @@ positions are implicit in record order, so consumers can rebuild offsets in their own string-index units. Single-value conversions (e.g. `头发` → `頭髮`) are not flagged. +The record stream is a machine-readable contract: every emitted line is +valid JSON, and the final `{"end"}` record only appears when the whole +input was processed. Input containing invalid UTF-8 aborts the stream with +an error (unlike plain conversion, which is byte-transparent), and a +missing `{"end"}` record means the stream is incomplete. + These modes are useful for diagnosing conversion issues: 1. Use `--segmentation` to verify that the input is segmented as expected. 2. Use `--inspect` to see which conversion stage produces an unexpected result. +3. Use `--ambiguities` to locate one-to-many conversions and resolve each + deduplicated `def` source to its candidates. Rules: -- `--segmentation` and `--inspect` are mutually exclusive. +- `--segmentation`, `--inspect`, and `--ambiguities` are mutually exclusive. ### Official / Recommended Ports diff --git a/node/cli.js b/node/cli.js index 1d6ee7dc0..c069e115a 100755 --- a/node/cli.js +++ b/node/cli.js @@ -57,6 +57,7 @@ Options: Unsupported in the npm CLI: --inspect Use the native OpenCC CLI for inspection output. --segmentation Use the native OpenCC CLI for segmentation output. + --ambiguities Use the native OpenCC CLI for ambiguity records. Built-in Configurations: ${BUILT_IN_CONFIGS.map(([name, description]) => ` ${name.padEnd(11)} ${description}`).join('\n')} @@ -123,7 +124,8 @@ function parseArgs(args) { options.output = readInlineOptionValue(arg, '--output'); } else if (arg === '--include-tofu-risk-dictionaries') { options.includeTofuRiskDictionaries = true; - } else if (arg === '--inspect' || arg === '--segmentation') { + } else if (arg === '--inspect' || arg === '--segmentation' || + arg === '--ambiguities') { throw new Error(`${arg} is not supported by the npm CLI. Use the native OpenCC CLI instead.`); } else if (arg === '--path' || arg.startsWith('--path=')) { throw new Error('--path is not supported by the npm CLI. Pass an explicit config file path instead.'); diff --git a/node/test.js b/node/test.js index 65fdea37d..4f704fd1d 100644 --- a/node/test.js +++ b/node/test.js @@ -434,6 +434,7 @@ describe('npm CLI', function () { assert.match(result.stdout, /Unsupported in the npm CLI:/); assert.match(result.stdout, /--inspect/); assert.match(result.stdout, /--segmentation/); + assert.match(result.stdout, /--ambiguities/); }); it('prints version', function () { @@ -456,6 +457,12 @@ describe('npm CLI', function () { }); assert.notEqual(segmentation.status, 0); assert.match(segmentation.stderr, /not supported/); + + const ambiguities = childProcess.spawnSync(process.execPath, [cli, '--ambiguities'], { + encoding: 'utf8', + }); + assert.notEqual(ambiguities.status, 0); + assert.match(ambiguities.stderr, /not supported/); }); it('rejects empty inline option values', function () { diff --git a/python/opencc/cli.py b/python/opencc/cli.py index 4e71896f8..457e7b164 100644 --- a/python/opencc/cli.py +++ b/python/opencc/cli.py @@ -48,7 +48,20 @@ def _build_parser(): return parser +_NATIVE_CLI_ONLY_FLAGS = ('--inspect', '--segmentation', '--ambiguities') + + def main(): + for arg in sys.argv[1:]: + flag = arg.split('=', 1)[0] + if flag in _NATIVE_CLI_ONLY_FLAGS: + print( + f'opencc: error: {flag} is not supported by the Python CLI. ' + 'Use the native OpenCC CLI instead.', + file=sys.stderr, + ) + sys.exit(1) + parser = _build_parser() args = parser.parse_args() diff --git a/python/tests/test_cli.py b/python/tests/test_cli.py index a6135fd15..8c638ec2a 100644 --- a/python/tests/test_cli.py +++ b/python/tests/test_cli.py @@ -121,6 +121,15 @@ def test_cli_include_tofu_risk_dictionaries_flag(tmp_path, monkeypatch): assert output_path.read_text(encoding='utf-8') == '𫝈' +def test_cli_rejects_native_only_modes(monkeypatch, capsys): + for flag in ('--inspect', '--segmentation', '--ambiguities'): + with pytest.raises(SystemExit) as exc_info: + _run_cli(monkeypatch, flag) + captured = capsys.readouterr() + assert exc_info.value.code == 1 + assert 'native OpenCC CLI' in captured.err + + def test_cli_rejects_same_input_and_output_file(tmp_path, monkeypatch, capsys): input_path = tmp_path / 'same.txt' input_path.write_text('汉字', encoding='utf-8') diff --git a/src/BUILD.bazel b/src/BUILD.bazel index 6fadb78cb..9156ad736 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -281,6 +281,19 @@ cc_library( deps = [ ":common_lib", ":segmentation_lib", + ":stream_window_lib", + ":utf8_util_lib", + ], +) + +# Shared streaming flush-window computation (private, non-installed header) +# used by ConverterStream and AmbiguityStream so both flush on identical +# boundaries. +cc_library( + name = "stream_window_lib", + hdrs = ["StreamWindow.hpp"], + visibility = ["//src:__pkg__"], + deps = [ ":utf8_util_lib", ], ) @@ -307,6 +320,7 @@ cc_library( ":dict_entry_lib", ":segmentation_lib", ":segments_lib", + ":stream_window_lib", ":utf8_util_lib", ], ) @@ -320,6 +334,7 @@ cc_test( ":conversion_lib", ":conversion_chain_lib", ":converter_lib", + ":dict_group_lib", ":lexicon_lib", ":max_match_segmentation_lib", ":pipeline_converter_lib", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 88046007b..7e99ab8ef 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -54,6 +54,7 @@ set( PrefixMatch.hpp SerializedValues.hpp SingleStageConverter.hpp + StreamWindow.hpp UTF8StringSlice.hpp ) diff --git a/src/ConversionAmbiguities.cpp b/src/ConversionAmbiguities.cpp index 63af28f42..2308def8a 100644 --- a/src/ConversionAmbiguities.cpp +++ b/src/ConversionAmbiguities.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -33,6 +34,7 @@ #include "Optional.hpp" #include "Segmentation.hpp" #include "Segments.hpp" +#include "StreamWindow.hpp" #include "UTF8Util.hpp" namespace opencc { @@ -47,6 +49,55 @@ struct Run { bool ambiguous; }; +// Prefix match with PrefixMatch's group semantics, via the Dict interface. +// The virtual Dict::MatchPrefix() cannot be called directly on groups: a +// DictGroup constructed with DictGroupMatchPolicy::Union reports Union via +// GetMatchPolicy() -- which PrefixMatch honors by taking the longest match +// across children -- but its inherited MatchPrefix() still short-circuits; +// only the UnionDictGroup subclass overrides it. Recursing on +// GetDictGroupItems()/GetMatchPolicy() applies the policy semantics +// uniformly, which also fixes plain Union-policy DictGroups nested inside +// other groups (where delegating to the parent's virtual MatchPrefix could +// not). +// +// The recursion is verified equivalent to both overrides (and to +// PrefixMatch's matcher tables): DictGroup::MatchPrefix returns the first +// child with any match, and UnionDictGroup::MatchPrefix takes the longest +// match across children with a strict > comparison so earlier children win +// ties -- exactly the two branches below. Neither override merges entry +// values across children for prefix lookup, so NumValues() of the returned +// entry is the correct candidate count. ConversionAmbiguitiesTest pins the +// longest-match and same-length-tie cases against Convert(). +Optional MatchPrefixLikeConversion(const Dict& dict, + const char* word, + size_t len) { + const std::list* items = dict.GetDictGroupItems(); + if (items == nullptr) { + return dict.MatchPrefix(word, len); + } + if (dict.GetMatchPolicy() == DictGroupMatchPolicy::ShortCircuit) { + for (const DictPtr& child : *items) { + const Optional match = + MatchPrefixLikeConversion(*child, word, len); + if (!match.IsNull()) { + return match; + } + } + return Optional::Null(); + } + Optional best = Optional::Null(); + for (const DictPtr& child : *items) { + const Optional match = + MatchPrefixLikeConversion(*child, word, len); + if (!match.IsNull() && + (best.IsNull() || + match.Get()->KeyLength() > best.Get()->KeyLength())) { + best = match; + } + } + return best; +} + // Replays Conversion::AppendConverted() for one dictionary over `in`, using // the DictEntry-returning MatchPrefix() so the candidate count is visible. // Appends the converted text to `out` and the alignment to `runs`. @@ -62,7 +113,8 @@ void WalkStage(const DictPtr& dict, std::string_view in, std::string* out, const char* const end = pstr + in.size(); while (pstr < end) { const size_t remaining = static_cast(end - pstr); - const Optional match = dict->MatchPrefix(pstr, remaining); + const Optional match = + MatchPrefixLikeConversion(*dict, pstr, remaining); if (match.IsNull()) { size_t charLen = UTF8Util::NextIdeographicDescriptionSequenceLength(pstr, remaining); @@ -124,23 +176,24 @@ std::vector Compose(const std::vector& a, const std::vector& b) { } out.push_back(Run{srcLen, dstLen, ambiguous}); } - // Trailing runs can only be zero-width on the shared middle coordinate - // (empty dictionary values / empty keys); fold them into the last group so - // both totals stay accounted for. + // Both sides measure the same intermediate text, so their totals match. + // Every b-side run consumes at least one intermediate byte (WalkStage + // advances by at least one byte per run), so b always exhausts with the + // main loop. The a-side, however, can trail with zero-width runs: an + // a-side run's width on the middle coordinate is its dstLen, i.e. the + // dictionary value's length, and an empty-value entry (constructible by + // library users, e.g. StrSingleValueDictEntry("x", "")) matching at the + // end of a segment yields dstLen == 0. Fold such trailing runs into the + // last group so the source side stays fully accounted for. + assert(ib == b.size()); for (; ia < a.size(); ia++) { + assert(a[ia].dstLen == 0); if (out.empty()) { out.push_back(Run{0, 0, false}); } out.back().srcLen += a[ia].srcLen; out.back().ambiguous = out.back().ambiguous || a[ia].ambiguous; } - for (; ib < b.size(); ib++) { - if (out.empty()) { - out.push_back(Run{0, 0, false}); - } - out.back().dstLen += b[ib].dstLen; - out.back().ambiguous = out.back().ambiguous || b[ib].ambiguous; - } return out; } @@ -167,15 +220,6 @@ AnnotatedConversion ConvertWithAmbiguities(const Converter& converter, text = normalized; } - SegmentsPtr segments; - const SegmentationPtr segmentation = converter.GetSegmentation(); - if (segmentation == nullptr) { - segments.reset(new Segments); - segments->AddSegment(text); - } else { - segments = segmentation->Segment(text); - } - std::unordered_map sourceIndexes; // Each segment is walked through the whole chain before moving to the // next (segment-outer, stage-inner), while ConversionChain::Convert is @@ -184,12 +228,7 @@ AnnotatedConversion ConvertWithAmbiguities(const Converter& converter, // matches across segment boundaries; if the chain ever gains cross- // segment optimizations, this walk must be restructured to keep the // byte-identical-output contract with Converter::Convert(). - for (const char* segment : *segments) { - // strlen() mirrors Convert(), which also consumes segments as - // NUL-terminated strings; input containing NUL bytes is truncated - // identically on both paths, so offsets stay aligned. - const std::string_view segmentView(segment, std::strlen(segment)); - + auto walkSegment = [&](std::string_view segmentView) { // Walk the segment through the chain, keeping the composed alignment // between the original segment and the current stage's output. std::vector aligned; @@ -227,6 +266,22 @@ AnnotatedConversion ConvertWithAmbiguities(const Converter& converter, dstOffset += run.dstLen; } result.output.append(current); + }; + + const SegmentationPtr segmentation = converter.GetSegmentation(); + if (segmentation == nullptr) { + // Mirror SingleStageConverter::Convert's no-segmentation branch, which + // converts the whole text as a single string_view without building a + // Segments object -- embedded NUL bytes are preserved on both paths. + walkSegment(text); + } else { + const SegmentsPtr segments = segmentation->Segment(text); + for (const char* segment : *segments) { + // strlen() mirrors Convert()'s segmented path, which also consumes + // segments as NUL-terminated strings; input containing NUL bytes is + // truncated identically on both paths, so offsets stay aligned. + walkSegment(std::string_view(segment, std::strlen(segment))); + } } return result; } @@ -258,6 +313,7 @@ AmbiguityStream::ConvertWindow(std::string_view window) { Chunk chunk; AnnotatedConversion result = ConvertWithAmbiguities(*impl->converter, window); + chunk.analyzed = result.analyzed; chunk.output = std::move(result.output); std::vector globalIndex(result.sources.size()); for (size_t i = 0; i < result.sources.size(); i++) { @@ -275,60 +331,24 @@ AmbiguityStream::ConvertWindow(std::string_view window) { return chunk; } -// Windowing below mirrors ConverterStream::ConvertChunk()/Finish() exactly; -// the withheld tail guarantees no match (and hence no ambiguous span) ever -// straddles a flush boundary. +// Windowing below shares internal::FlushableByteCount with +// ConverterStream::ConvertChunk, so both wrappers flush on identical +// boundaries; the withheld tail guarantees no match (and hence no ambiguous +// span) ever straddles a flush boundary. AmbiguityStream::Chunk AmbiguityStream::ConvertChunk(std::string_view input) { std::string& pending = impl->pending; if (!input.empty()) { pending.append(input); } - if (pending.empty()) { - return Chunk(); - } - - const char* bufferBegin = pending.data(); - const char* bufferEnd = bufferBegin + pending.size(); - const char* completeEnd = bufferBegin; - while (completeEnd < bufferEnd) { - const size_t nextCharLen = UTF8Util::NextCharLength(completeEnd); - if (completeEnd + nextCharLen > bufferEnd) { - break; - } - completeEnd += nextCharLen; - } - - const char* keepStart = completeEnd; - size_t charsKept = 0; - while (keepStart > bufferBegin && charsKept < impl->maxKeepChars) { - const size_t prevCharLen = UTF8Util::PrevCharLength(keepStart); - keepStart -= prevCharLen; - charsKept++; - } - - const char* idsKeepStart = completeEnd; - const char* idsCandidate = completeEnd; - size_t idsCharsScanned = 0; - const size_t kMaxIDSCodePoints = 64; - while (idsCandidate > bufferBegin && idsCharsScanned < kMaxIDSCodePoints) { - idsCandidate -= UTF8Util::PrevCharLength(idsCandidate); - idsCharsScanned++; - if (UTF8Util::IsIncompleteIdeographicDescriptionSequencePrefix( - idsCandidate, completeEnd - idsCandidate)) { - idsKeepStart = idsCandidate; - } - } - if (idsKeepStart < keepStart) { - keepStart = idsKeepStart; - } - - if (keepStart == bufferBegin) { + const size_t flushable = + internal::FlushableByteCount(pending, impl->maxKeepChars); + if (flushable == 0) { return Chunk(); } - Chunk chunk = ConvertWindow(std::string_view( - bufferBegin, static_cast(keepStart - bufferBegin))); - pending.erase(0, static_cast(keepStart - bufferBegin)); + Chunk chunk = + ConvertWindow(std::string_view(pending.data(), flushable)); + pending.erase(0, flushable); return chunk; } diff --git a/src/ConversionAmbiguities.hpp b/src/ConversionAmbiguities.hpp index 552806639..1e36b23aa 100644 --- a/src/ConversionAmbiguities.hpp +++ b/src/ConversionAmbiguities.hpp @@ -26,6 +26,7 @@ #include "Common.hpp" #include "Converter.hpp" #include "Export.hpp" +#include "StreamWindow.hpp" namespace opencc { @@ -98,17 +99,6 @@ struct AnnotatedConversion { OPENCC_EXPORT AnnotatedConversion ConvertWithAmbiguities(const Converter& converter, std::string_view text); -/** - * Default keep-tail window for AmbiguityStream, in Unicode code points. - * Must equal ConverterStream's default maxKeepChars (Converter.hpp) so the - * two wrappers flush on identical boundaries; the constant lives in this - * private header instead of the installed one to keep this feature free of - * installed-header changes. ConversionAmbiguitiesTest pins the two - * defaults together behaviorally (identical per-chunk flush output), so a - * drift fails tests instead of silently desynchronizing the streams. - */ -inline constexpr size_t kAmbiguityStreamDefaultKeepChars = 16; - /** * Streaming variant of ConvertWithAmbiguities() with bounded memory. * @@ -130,10 +120,15 @@ class OPENCC_EXPORT AmbiguityStream { std::string output; std::vector ambiguities; std::vector newSources; + /** Mirrors AnnotatedConversion::analyzed: false when the underlying + * converter could not be analyzed (no single conversion chain), in + * which case empty ambiguities means "unknown", not "none". */ + bool analyzed = true; }; - explicit AmbiguityStream(ConverterPtr converter, - size_t maxKeepChars = kAmbiguityStreamDefaultKeepChars); + explicit AmbiguityStream( + ConverterPtr converter, + size_t maxKeepChars = internal::kDefaultStreamKeepChars); ~AmbiguityStream(); /** Appends @p input and flushes everything but the kept tail. */ diff --git a/src/ConversionAmbiguitiesTest.cpp b/src/ConversionAmbiguitiesTest.cpp index edaab8f26..5a8733def 100644 --- a/src/ConversionAmbiguitiesTest.cpp +++ b/src/ConversionAmbiguitiesTest.cpp @@ -21,6 +21,7 @@ #include "Conversion.hpp" #include "ConversionChain.hpp" #include "Converter.hpp" +#include "DictGroup.hpp" #include "Lexicon.hpp" #include "MaxMatchSegmentation.hpp" #include "PipelineConverter.hpp" @@ -153,6 +154,120 @@ TEST_F(ConversionAmbiguitiesTest, StraddlingMatchCoarsensSpan) { EXPECT_EQ("ab", result.sources[result.ambiguities[0].sourceIndex]); } +TEST_F(ConversionAmbiguitiesTest, UnionPolicyDictGroupMatchesConvert) { + // A DictGroup constructed directly with the Union policy reports Union + // via GetMatchPolicy() (honored by Convert()'s PrefixMatch tables), but + // its inherited MatchPrefix() short-circuits; the walk must apply the + // policy semantics itself or its output diverges from Convert(). + const DictPtr group(new DictGroup( + std::list{MakeDict({{"a", {"x"}}}), MakeDict({{"ab", {"y"}}})}, + DictGroupMatchPolicy::Union)); + std::list conversions{ConversionPtr(new Conversion(group))}; + const ConverterPtr converter(new SingleStageConverter( + SegmentationPtr(new MaxMatchSegmentation(group)), + ConversionChainPtr(new ConversionChain(conversions)))); + ASSERT_EQ("y", converter->Convert("ab")); + const AnnotatedConversion result = ConvertWithAmbiguities(*converter, "ab"); + EXPECT_EQ(converter->Convert("ab"), result.output); +} + +TEST_F(ConversionAmbiguitiesTest, UnionPolicyTieBreakMatchesConvert) { + // Two children share the same key with different values; PrefixMatch's + // union semantics let the earlier child win the tie, and the walk's + // recursion (strict > comparison) must agree with Convert(). + const DictPtr group(new DictGroup( + std::list{MakeDict({{"ab", {"first"}}}), + MakeDict({{"ab", {"second"}}})}, + DictGroupMatchPolicy::Union)); + std::list conversions{ConversionPtr(new Conversion(group))}; + const ConverterPtr converter(new SingleStageConverter( + SegmentationPtr(new MaxMatchSegmentation(group)), + ConversionChainPtr(new ConversionChain(conversions)))); + const AnnotatedConversion result = ConvertWithAmbiguities(*converter, "ab"); + EXPECT_EQ(converter->Convert("ab"), result.output); + EXPECT_EQ("first", result.output); +} + +TEST_F(ConversionAmbiguitiesTest, UnionNumValuesComesFromWinningChild) { + // The flag must reflect NumValues() of exactly the entry the union + // picked (first child wins same-length ties), not values merged across + // children. Uses the UnionDictGroup subclass directly so the override + // path is covered alongside the plain Union-policy DictGroup tests. + auto makeConverter = [](const DictPtr& first, const DictPtr& second) { + const DictPtr group( + new UnionDictGroup(std::list{first, second})); + std::list conversions{ + ConversionPtr(new Conversion(group))}; + return ConverterPtr(new SingleStageConverter( + SegmentationPtr(new MaxMatchSegmentation(group)), + ConversionChainPtr(new ConversionChain(conversions)))); + }; + const DictPtr multi = MakeDict({{"ab", {"x", "y"}}}); + const DictPtr single = MakeDict({{"ab", {"z"}}}); + + // Multi-value child wins the tie: flagged, default is its first value. + const AnnotatedConversion flagged = + ConvertWithAmbiguities(*makeConverter(multi, single), "ab"); + EXPECT_EQ("x", flagged.output); + ASSERT_EQ(1u, flagged.ambiguities.size()); + EXPECT_EQ("ab", flagged.sources[flagged.ambiguities[0].sourceIndex]); + + // Single-value child wins the tie: NOT flagged, even though another + // child holds a multi-value entry for the same key. + const AnnotatedConversion unflagged = + ConvertWithAmbiguities(*makeConverter(single, multi), "ab"); + EXPECT_EQ("z", unflagged.output); + EXPECT_TRUE(unflagged.ambiguities.empty()); +} + +TEST_F(ConversionAmbiguitiesTest, EmptyDictionaryValueDoesNotBreakCompose) { + // An empty-value entry matching at the end of a segment leaves a + // trailing zero-width run on the alignment's middle coordinate; Compose + // must fold it instead of asserting or reading out of bounds, and the + // output must still match Convert(). + std::list conversions{ + ConversionPtr( + new Conversion(MakeDict({{"a", {"m", "n"}}, {"b", {""}}}))), + ConversionPtr(new Conversion(MakeDict({{"m", {"M"}}})))}; + const ConverterPtr converter(new SingleStageConverter( + SegmentationPtr(new MaxMatchSegmentation(MakeDict({{"ab", {"ab"}}}))), + ConversionChainPtr(new ConversionChain(conversions)))); + const AnnotatedConversion result = ConvertWithAmbiguities(*converter, "ab"); + EXPECT_EQ(converter->Convert("ab"), result.output); + EXPECT_EQ("M", result.output); + ASSERT_EQ(1u, result.ambiguities.size()); + // The trailing fold widens the span's source to cover the vanished tail. + EXPECT_EQ("ab", result.sources[result.ambiguities[0].sourceIndex]); +} + +TEST_F(ConversionAmbiguitiesTest, NulBytePreservedWithoutSegmentation) { + // Without a segmentation step, Convert() processes the whole text as one + // string_view, preserving embedded NUL bytes; the walk must do the same + // instead of truncating at the first NUL. + std::list conversions{ + ConversionPtr(new Conversion(MakeDict({{"b", {"y", "z"}}})))}; + const ConverterPtr converter(new SingleStageConverter( + nullptr, ConversionChainPtr(new ConversionChain(conversions)))); + const std::string input("a\0b", 3); + const std::string expected = converter->Convert(input); + ASSERT_EQ(std::string("a\0y", 3), expected); + const AnnotatedConversion result = ConvertWithAmbiguities(*converter, input); + EXPECT_EQ(expected, result.output); + ASSERT_EQ(1u, result.ambiguities.size()); + EXPECT_EQ(2u, result.ambiguities[0].outputOffset); + EXPECT_EQ("b", result.sources[result.ambiguities[0].sourceIndex]); +} + +TEST_F(ConversionAmbiguitiesTest, StreamChunkCarriesAnalyzedFlag) { + const ConverterPtr pipeline(new PipelineConverter( + {MakeConverter({MakeDict({{"b", {"y", "z"}}})})})); + AmbiguityStream unanalyzed(pipeline); + EXPECT_FALSE(unanalyzed.Finish("b").analyzed); + + AmbiguityStream analyzed(MakeConverter({MakeDict({{"b", {"y", "z"}}})})); + EXPECT_TRUE(analyzed.Finish("b").analyzed); +} + TEST_F(ConversionAmbiguitiesTest, PipelineConverterIsFlaggedUnanalyzed) { // A converter without a single conversion chain still converts, but the // result must be distinguishable from "converted with no ambiguities". diff --git a/src/Converter.cpp b/src/Converter.cpp index c7b8109fc..0fbe44c34 100644 --- a/src/Converter.cpp +++ b/src/Converter.cpp @@ -17,7 +17,7 @@ */ #include "Converter.hpp" -#include "UTF8Util.hpp" +#include "StreamWindow.hpp" using namespace opencc; @@ -25,52 +25,15 @@ std::string ConverterStream::ConvertChunk(std::string_view input) { if (!input.empty()) { pending.append(input); } - if (pending.empty()) { + const size_t flushable = + internal::FlushableByteCount(pending, maxKeepChars); + if (flushable == 0) { return std::string(); } - const char* bufferBegin = pending.data(); - const char* bufferEnd = bufferBegin + pending.size(); - const char* completeEnd = bufferBegin; - while (completeEnd < bufferEnd) { - const size_t nextCharLen = UTF8Util::NextCharLength(completeEnd); - if (completeEnd + nextCharLen > bufferEnd) { - break; - } - completeEnd += nextCharLen; - } - - const char* keepStart = completeEnd; - size_t charsKept = 0; - while (keepStart > bufferBegin && charsKept < maxKeepChars) { - const size_t prevCharLen = UTF8Util::PrevCharLength(keepStart); - keepStart -= prevCharLen; - charsKept++; - } - - const char* idsKeepStart = completeEnd; - const char* idsCandidate = completeEnd; - size_t idsCharsScanned = 0; - const size_t kMaxIDSCodePoints = 64; - while (idsCandidate > bufferBegin && idsCharsScanned < kMaxIDSCodePoints) { - idsCandidate -= UTF8Util::PrevCharLength(idsCandidate); - idsCharsScanned++; - if (UTF8Util::IsIncompleteIdeographicDescriptionSequencePrefix( - idsCandidate, completeEnd - idsCandidate)) { - idsKeepStart = idsCandidate; - } - } - if (idsKeepStart < keepStart) { - keepStart = idsKeepStart; - } - - if (keepStart == bufferBegin) { - return std::string(); - } - - const std::string output = converter->Convert(std::string_view( - bufferBegin, static_cast(keepStart - bufferBegin))); - pending.erase(0, static_cast(keepStart - bufferBegin)); + const std::string output = + converter->Convert(std::string_view(pending.data(), flushable)); + pending.erase(0, flushable); return output; } diff --git a/src/StreamWindow.hpp b/src/StreamWindow.hpp new file mode 100644 index 000000000..826aca43d --- /dev/null +++ b/src/StreamWindow.hpp @@ -0,0 +1,97 @@ +/* + * Open Chinese Convert + * + * Copyright 2010-2026 Carbo Kuo and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "UTF8Util.hpp" + +namespace opencc { +namespace internal { + +/** + * Default keep-tail window for streaming converters, in Unicode code + * points. Must equal ConverterStream's default maxKeepChars + * (Converter.hpp, an installed header that cannot reference this private + * one); ConversionAmbiguitiesTest pins the two defaults together + * behaviorally, so a drift fails tests. + */ +inline constexpr size_t kDefaultStreamKeepChars = 16; + +/** + * Returns the number of bytes at the front of @p pending that a streaming + * converter may flush now, or 0 if everything must be kept. + * + * The boundary excludes a trailing incomplete UTF-8 sequence, then withholds + * the last @p maxKeepChars complete code points (so a phrase straddling two + * consecutive chunks is not split across separate conversions), extending the + * kept tail further if it ends in an incomplete ideographic description + * sequence prefix. + * + * Shared by ConverterStream and AmbiguityStream so that both always flush on + * identical boundaries; a windowing fix applied here reaches every streaming + * wrapper at once. Private (non-installed) header. + */ +inline size_t FlushableByteCount(std::string_view pending, + size_t maxKeepChars) { + if (pending.empty()) { + return 0; + } + + const char* bufferBegin = pending.data(); + const char* bufferEnd = bufferBegin + pending.size(); + const char* completeEnd = bufferBegin; + while (completeEnd < bufferEnd) { + const size_t nextCharLen = UTF8Util::NextCharLength(completeEnd); + if (completeEnd + nextCharLen > bufferEnd) { + break; + } + completeEnd += nextCharLen; + } + + const char* keepStart = completeEnd; + size_t charsKept = 0; + while (keepStart > bufferBegin && charsKept < maxKeepChars) { + const size_t prevCharLen = UTF8Util::PrevCharLength(keepStart); + keepStart -= prevCharLen; + charsKept++; + } + + const char* idsKeepStart = completeEnd; + const char* idsCandidate = completeEnd; + size_t idsCharsScanned = 0; + const size_t kMaxIDSCodePoints = 64; + while (idsCandidate > bufferBegin && idsCharsScanned < kMaxIDSCodePoints) { + idsCandidate -= UTF8Util::PrevCharLength(idsCandidate); + idsCharsScanned++; + if (UTF8Util::IsIncompleteIdeographicDescriptionSequencePrefix( + idsCandidate, completeEnd - idsCandidate)) { + idsKeepStart = idsCandidate; + } + } + if (idsKeepStart < keepStart) { + keepStart = idsKeepStart; + } + + return static_cast(keepStart - bufferBegin); +} + +} // namespace internal +} // namespace opencc diff --git a/src/tools/CommandLineMain.cpp b/src/tools/CommandLineMain.cpp index da8642b63..61518fe87 100644 --- a/src/tools/CommandLineMain.cpp +++ b/src/tools/CommandLineMain.cpp @@ -401,42 +401,75 @@ std::string ConvertLineByMode(const std::string& line) { // {"lit":""} literal (unambiguous) output run // {"amb":{"t":"","s":N}} ambiguous span; N is a global source index // {"end":{...}} stream summary -void WriteAmbiguityChunkRecords(const AmbiguityStream::Chunk& chunk, - FILE* fout) { +// Returns the number of bytes written to fout, so callers can account for +// the actual emitted size (record framing and JSON escaping included). +// +// Unlike plain conversion (byte-transparent) and --inspect (human-oriented), +// this stream is a machine-readable contract, so the writer validates +// encoding: length-based walking tolerates multi-byte lead bytes with +// invalid continuation bytes, and without validation such input would flow +// verbatim into the records and break every strict JSON consumer. A +// validation failure aborts the stream loudly (no end record, error exit) +// instead of emitting broken JSON. +using ValidatingJsonWriter = + rapidjson::Writer, + rapidjson::UTF8<>, rapidjson::CrtAllocator, + rapidjson::kWriteValidateEncodingFlag>; + +size_t WriteAmbiguityChunkRecords(const AmbiguityStream::Chunk& chunk, + FILE* fout) { rapidjson::StringBuffer buffer; - auto flushRecord = [&buffer, fout]() { + size_t bytesWritten = 0; + // fputs() is safe even when the converted text contains NUL bytes (the + // no-segmentation walk preserves them): rapidjson escapes control + // characters as backslash-u0000 escapes, so the serialized buffer + // never holds a raw NUL. Keep that property in mind before switching + // the writer or the emission to anything that does not escape control + // characters. + auto flushRecord = [&buffer, fout, &bytesWritten]() { fputs(buffer.GetString(), fout); fputc('\n', fout); + bytesWritten += buffer.GetSize() + 1; buffer.Clear(); }; + auto writeValidatedString = [](ValidatingJsonWriter& writer, + const char* data, size_t length) { + if (!writer.String(data, length)) { + throw Exception( + "Converted output contains invalid UTF-8 (from the input or a " + "dictionary); --ambiguities emits machine-readable JSON and " + "cannot represent it. Aborting record stream."); + } + }; for (const std::string& source : chunk.newSources) { - rapidjson::Writer writer(buffer); + ValidatingJsonWriter writer(buffer); writer.StartObject(); writer.Key("def"); - writer.String(source.c_str(), source.size()); + writeValidatedString(writer, source.c_str(), source.size()); writer.EndObject(); flushRecord(); } size_t consumed = 0; auto writeLiteral = [&](size_t until) { if (until > consumed) { - rapidjson::Writer writer(buffer); + ValidatingJsonWriter writer(buffer); writer.StartObject(); writer.Key("lit"); - writer.String(chunk.output.c_str() + consumed, until - consumed); + writeValidatedString(writer, chunk.output.c_str() + consumed, + until - consumed); writer.EndObject(); flushRecord(); } }; for (const auto& span : chunk.ambiguities) { writeLiteral(span.outputOffset); - rapidjson::Writer writer(buffer); + ValidatingJsonWriter writer(buffer); writer.StartObject(); writer.Key("amb"); writer.StartObject(); writer.Key("t"); - writer.String(chunk.output.c_str() + span.outputOffset, - span.outputLength); + writeValidatedString(writer, chunk.output.c_str() + span.outputOffset, + span.outputLength); writer.Key("s"); writer.Uint64(span.sourceIndex); writer.EndObject(); @@ -445,6 +478,7 @@ void WriteAmbiguityChunkRecords(const AmbiguityStream::Chunk& chunk, consumed = span.outputOffset + span.outputLength; } writeLiteral(chunk.output.size()); + return bytesWritten; } // Streaming --ambiguities over any input stream (file or stdin): bounded @@ -455,14 +489,17 @@ void ConvertAmbiguitiesStream(FILE* fin, FILE* fout) { const int BUFFER_SIZE = 1024 * 1024; std::string buffer(BUFFER_SIZE, '\0'); AmbiguityStream stream(converter); - size_t outputBytes = 0; + // convertedBytes counts the underlying converted text (reported in the + // end record); measurement.outputBytes gets the bytes actually written, + // consistent with every other output mode. + size_t convertedBytes = 0; size_t ambiguityCount = 0; auto emitChunk = [&](const AmbiguityStream::Chunk& chunk) { - outputBytes += chunk.output.size(); + convertedBytes += chunk.output.size(); ambiguityCount += chunk.ambiguities.size(); const auto writeStart = std::chrono::steady_clock::now(); - WriteAmbiguityChunkRecords(chunk, fout); + measurement.outputBytes += WriteAmbiguityChunkRecords(chunk, fout); if (!noFlush) { fflush(fout); } @@ -471,9 +508,11 @@ void ConvertAmbiguitiesStream(FILE* fin, FILE* fout) { }; bool finished = false; + bool readError = false; while (!feof(fin)) { size_t length = fread(&buffer[0], sizeof(char), buffer.size(), fin); if (length == 0) { + readError = ferror(fin) != 0; break; } measurement.inputBytes += length; @@ -492,6 +531,13 @@ void ConvertAmbiguitiesStream(FILE* fin, FILE* fout) { break; } } + if (readError) { + // Do NOT emit the end record: it is the stream's integrity signal, and + // emitting it after a failed read would falsely mark a truncated stream + // as complete. The missing end record plus the error exit tell the + // consumer the stream is incomplete. + throw Exception("Error reading input stream in --ambiguities mode."); + } if (!finished) { const auto convertStart = std::chrono::steady_clock::now(); const AmbiguityStream::Chunk chunk = stream.Finish(); @@ -499,7 +545,6 @@ void ConvertAmbiguitiesStream(FILE* fin, FILE* fout) { std::chrono::steady_clock::now() - convertStart); emitChunk(chunk); } - measurement.outputBytes += outputBytes; rapidjson::StringBuffer endBuffer; rapidjson::Writer writer(endBuffer); @@ -507,18 +552,22 @@ void ConvertAmbiguitiesStream(FILE* fin, FILE* fout) { writer.Key("end"); writer.StartObject(); writer.Key("output_bytes"); - writer.Uint64(outputBytes); + writer.Uint64(convertedBytes); writer.Key("ambiguities"); writer.Uint64(ambiguityCount); writer.Key("sources"); writer.Uint64(stream.SourceCount()); writer.EndObject(); writer.EndObject(); + const auto writeStart = std::chrono::steady_clock::now(); fputs(endBuffer.GetString(), fout); fputc('\n', fout); + measurement.outputBytes += endBuffer.GetSize() + 1; if (!noFlush) { fflush(fout); } + measurement.writeMs += DurationToMilliseconds( + std::chrono::steady_clock::now() - writeStart); } void ConvertStream(FILE* fin, FILE* fout) { @@ -598,11 +647,12 @@ void ConvertLineByLine() { void ConvertFileStreams(FILE* fin, FILE* fout) { try { if (outputMode == OutputMode::Ambiguities) { + // Falls through to the shared fclose epilogue below; an early return + // here would leak both streams and break --in-place, which replaces + // the output file after this function and requires it to be closed. ConvertAmbiguitiesStream(fin, fout); - return; - } - if (outputMode == OutputMode::Segmentation || - outputMode == OutputMode::Inspect) { + } else if (outputMode == OutputMode::Segmentation || + outputMode == OutputMode::Inspect) { // Inspect/segmentation modes process line by line using // std::getline to handle arbitrarily long lines. bool isFirstLine = true; diff --git a/test/CommandLineConvertTest.cpp b/test/CommandLineConvertTest.cpp index deceb24f5..8736085b7 100644 --- a/test/CommandLineConvertTest.cpp +++ b/test/CommandLineConvertTest.cpp @@ -16,6 +16,7 @@ * limitations under the License. */ +#include #include #include #include @@ -484,8 +485,8 @@ TEST_F(CommandLineConvertTest, AmbiguitiesEmitsDefineOnFirstUseRecords) { std::istringstream records(GetFileContents(recordsFile)); std::string line; std::string reconstructed; - size_t defs = 0; - size_t ambs = 0; + std::vector defs; + std::vector ambIndexes; bool sawEnd = false; while (std::getline(records, line)) { if (line.empty()) { @@ -498,32 +499,42 @@ TEST_F(CommandLineConvertTest, AmbiguitiesEmitsDefineOnFirstUseRecords) { ASSERT_TRUE(doc.IsObject()) << line; if (doc.HasMember("def")) { ASSERT_TRUE(doc["def"].IsString()); - defs++; + defs.push_back(doc["def"].GetString()); } else if (doc.HasMember("lit")) { ASSERT_TRUE(doc["lit"].IsString()); reconstructed += doc["lit"].GetString(); } else if (doc.HasMember("amb")) { ASSERT_TRUE(doc["amb"].IsObject()); + ASSERT_TRUE(doc["amb"].HasMember("t") && doc["amb"]["t"].IsString()) + << line; + ASSERT_TRUE(doc["amb"].HasMember("s") && doc["amb"]["s"].IsUint64()) + << line; // Define-on-first-use contract: every reference points at an // already-defined source index. - ASSERT_LT(doc["amb"]["s"].GetUint64(), defs) << line; + ASSERT_LT(doc["amb"]["s"].GetUint64(), defs.size()) << line; reconstructed += doc["amb"]["t"].GetString(); - ambs++; + ambIndexes.push_back(doc["amb"]["s"].GetUint64()); } else if (doc.HasMember("end")) { sawEnd = true; EXPECT_EQ(reconstructed.size(), doc["end"]["output_bytes"].GetUint64()); - EXPECT_EQ(ambs, doc["end"]["ambiguities"].GetUint64()); - EXPECT_EQ(defs, doc["end"]["sources"].GetUint64()); + EXPECT_EQ(ambIndexes.size(), doc["end"]["ambiguities"].GetUint64()); + EXPECT_EQ(defs.size(), doc["end"]["sources"].GetUint64()); } else { FAIL() << "unknown record: " << line; } } EXPECT_TRUE(sawEnd); - EXPECT_EQ(2u, ambs); - EXPECT_EQ(1u, defs); // Interleaved records reconstruct exactly the plain conversion output. EXPECT_EQ(GetFileContents(convertFile), reconstructed); + // 文丑 appears twice in the input: dedup means exactly one def, and both + // occurrences reference it. Invariant-style, so unrelated dictionary + // updates cannot break this test. + ASSERT_EQ(1, std::count(defs.begin(), defs.end(), "文丑")); + const uint64_t wenchouIndex = static_cast( + std::find(defs.begin(), defs.end(), "文丑") - defs.begin()); + EXPECT_GE(std::count(ambIndexes.begin(), ambIndexes.end(), wenchouIndex), + 2); } TEST_F(CommandLineConvertTest, AmbiguitiesCoversMultiStageChain) { @@ -571,8 +582,87 @@ TEST_F(CommandLineConvertTest, AmbiguitiesCoversMultiStageChain) { } } EXPECT_EQ(GetFileContents(convertFile), reconstructed); - EXPECT_EQ((std::vector{"下面", "信号"}), defs); - EXPECT_EQ((std::vector{"下面", "信号"}), ambSources); + // Contains-style assertions so unrelated dictionary updates (new + // multi-value entries elsewhere in the sentence) cannot break the test. + EXPECT_NE(defs.end(), std::find(defs.begin(), defs.end(), "下面")); + EXPECT_NE(defs.end(), std::find(defs.begin(), defs.end(), "信号")); + EXPECT_NE(ambSources.end(), + std::find(ambSources.begin(), ambSources.end(), "下面")); + EXPECT_NE(ambSources.end(), + std::find(ambSources.begin(), ambSources.end(), "信号")); +} + +TEST_F(CommandLineConvertTest, AmbiguitiesRejectsInvalidUtf8) { + // Length-based walking tolerates a multi-byte lead byte with invalid + // continuation bytes; the record writer validates encoding so such input + // aborts the stream (non-zero exit, no end record) instead of emitting + // JSON that strict consumers reject. Plain conversion stays + // byte-transparent and is unaffected. + const std::string inputFile = InputFile("ambiguities_invalid_utf8"); + const std::string recordsFile = OutputFile("ambiguities_invalid_utf8"); + const std::string convertFile = OutputFile("ambiguities_invalid_utf8_conv"); + + { + std::ofstream ofs(inputFile, std::ios::binary); + ASSERT_TRUE(ofs.is_open()); + ofs << "abc\xE4\x41\x41zzz"; + } + + // The same input converts fine in byte-transparent convert mode, pinning + // the failure below to the record writer's encoding validation rather + // than some earlier stage. + ASSERT_EQ(0, RunCommand(TestCommand("s2t", inputFile, convertFile))); + ASSERT_NE(0, RunCommand(TestCommandWithFlags("s2t", inputFile, recordsFile, + "--ambiguities"))); + EXPECT_EQ(std::string::npos, GetFileContents(recordsFile).find("\"end\"")); +} + +TEST_F(CommandLineConvertTest, AmbiguitiesInPlaceInvalidUtf8KeepsOriginal) { + // The record writer throws on invalid UTF-8; with --in-place the + // exception must abort before the temp-file replacement so the user's + // original file survives intact, and the temp file must be cleaned up. + // A dedicated directory makes the no-residue assertion exact. + const fs::path dir = + fs::u8path(OutputDirectory()) / fs::u8path("ambiguities_inplace_bad"); + fs::create_directories(dir); + const fs::path file = dir / fs::u8path("input.txt"); + const std::string original = "abc\xE4\x41\x41zzz"; + + { + std::ofstream ofs(file, std::ios::binary); + ASSERT_TRUE(ofs.is_open()); + ofs << original; + } + + ASSERT_NE(0, RunCommand(TestCommand("s2t", file.u8string(), + file.u8string(), "", + "--ambiguities --in-place"))); + EXPECT_EQ(original, GetFileContents(file)); + size_t entries = 0; + for (const auto& entry : fs::directory_iterator(dir)) { + (void)entry; + entries++; + } + EXPECT_EQ(1u, entries) << "temporary file left behind"; +} + +TEST_F(CommandLineConvertTest, AmbiguitiesInPlaceRewritesFile) { + // Regression: the --ambiguities branch used to early-return from + // ConvertFileStreams, skipping the fclose epilogue; --in-place then + // replaced the output file while both streams were still open (fails on + // Windows, risks unflushed data elsewhere). + const std::string file = OutputFile("ambiguities_inplace"); + { + std::ofstream ofs(file, std::ios::binary); + ASSERT_TRUE(ofs.is_open()); + ofs << "文丑"; + } + + ASSERT_EQ(0, RunCommand( + TestCommand("s2t", file, file, "", "--ambiguities --in-place"))); + const std::string contents = GetFileContents(file); + EXPECT_NE(std::string::npos, contents.find("{\"def\":\"文丑\"}")) << contents; + EXPECT_NE(std::string::npos, contents.find("\"end\"")) << contents; } TEST_F(CommandLineConvertTest, StdinPreservesLineEndingsAndUnknownCharacters) {