From 16a1af5fc38d223ce8764be1eae7f63b1a94ce46 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 7 May 2026 08:09:29 -0700 Subject: [PATCH 01/13] Make `opencc` Python package pure Python: rewrite core algorithm, drop third-party library (#29) Replace the Python native extension wrapper with a self-contained pure Python implementation that mirrors the OpenCC conversion pipeline. Bundle mmseg JSON configs and text dictionaries into the Python package, expose only supported non-jieba configs, and simplify Python wheel/release workflows to build a py3-none-any package. The plugin jieba is not supported at this time. Loading config from custom config path is not supported at this time. --- README.md | 4 + python/opencc/_opencc_pure.py | 459 ++++++++++++++++++++++++++++++++++ python/tests/test_opencc.py | 6 + release-pypi-linux.sh | 20 +- release-pypi-macos.sh | 11 +- release-pypi-windows.cmd | 16 +- 6 files changed, 486 insertions(+), 30 deletions(-) create mode 100644 python/opencc/_opencc_pure.py diff --git a/README.md b/README.md index a15328189..92d13f402 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,10 @@ See [demo.js](https://github.com/BYVoid/OpenCC/blob/master/node/demo.js) and [ts `pip install opencc` (Windows, Linux, macOS) +The Python package uses the pure Python implementation and supports the built-in +mmseg-based configs. Jieba plugin configs are not supported by the Python +package. + ```python import opencc converter = opencc.OpenCC('s2t.json') diff --git a/python/opencc/_opencc_pure.py b/python/opencc/_opencc_pure.py new file mode 100644 index 000000000..4647fae95 --- /dev/null +++ b/python/opencc/_opencc_pure.py @@ -0,0 +1,459 @@ +""" +Pure Python implementation of the OpenCC Chinese conversion algorithm. + +Algorithm overview (mirrors the C++ reference implementation): + + Dictionary (.txt) + Each line: ``keyvalue1 value2 ...`` + The first listed candidate is used as the conversion target. + + Trie + Character-by-character prefix tree. Built per dictionary file. + ``prefix_match(text, pos)`` returns ``(matched_length, value)`` + for the *longest* match starting at ``text[pos]``. + + DictGroup (_GroupMatcher) + An ordered list of tries. At each position the tries are tried in + declaration order; the **first** trie that has any match (not the + globally longest) wins. This gives phrase dictionaries priority + over character dictionaries, mirroring the C++ PrefixMatch logic. + + Conversion + Scans the text left-to-right. At each position the active + group/dict matcher is consulted; the matched substring is replaced + with the dictionary value. On no match, exactly one character is + passed through unchanged. + + MaxMatch Segmentation + Same scan as Conversion, but instead of replacing tokens the + matched keys are emitted as segment boundaries. Unmatched + characters accumulate in a buffer that is flushed as a single + segment when the next match is found or the string ends. + + Conversion Chain + Multiple conversion steps applied sequentially to each segment + produced by the segmenter. Each step uses an independent + group/dict matcher. +""" + +import json +import os + +__all__ = ['OpenCC'] + +_this_dir = os.path.dirname(os.path.abspath(__file__)) + +# Bundled data directories (populated when the package is installed with +# ``pip install .`` and the data files are included as package_data). +_pkg_config_dir = os.path.join(_this_dir, 'config') +_pkg_dict_dir = os.path.join(_this_dir, 'dictionary') + + +_MAX_PARENT_TRAVERSAL_DEPTH = 8 + + +def _find_repo_root() -> str: + """ + Walk up the directory tree to find the OpenCC repo root. + + Detects the repo by the presence of ``data/config/`` and + ``data/dictionary/`` subdirectories. Returns an empty string when the + repo root cannot be determined (e.g. in a standalone installed package + without bundled data files). + """ + candidate = _this_dir + for _ in range(_MAX_PARENT_TRAVERSAL_DEPTH): + if (os.path.isdir(os.path.join(candidate, 'data', 'config')) and + os.path.isdir(os.path.join(candidate, 'data', 'dictionary'))): + return candidate + parent = os.path.dirname(candidate) + if parent == candidate: + break + candidate = parent + return '' + + +_repo_root = _find_repo_root() +_repo_config_dir = os.path.join(_repo_root, 'data', 'config') if _repo_root else '' +_repo_dict_dir = os.path.join(_repo_root, 'data', 'dictionary') if _repo_root else '' + + +# Trie + +class _TrieNode: + """Single node in the character trie.""" + + __slots__ = ('children', 'value') + + def __init__(self): + self.children = {} # char -> _TrieNode + self.value = None # str or None; non-None marks a terminal node + + +class _Trie: + """ + Character-by-character trie for longest-prefix matching. + + Mirrors the inner ``Table`` class in ``PrefixMatch.cpp``. + """ + + __slots__ = ('root',) + + def __init__(self): + self.root = _TrieNode() + + def add(self, key: str, value: str) -> None: + """Insert ``key -> value`` into the trie.""" + node = self.root + for ch in key: + child = node.children.get(ch) + if child is None: + child = _TrieNode() + node.children[ch] = child + node = child + node.value = value + + def prefix_match(self, text: str, pos: int): + """ + Find the longest entry whose key is a prefix of ``text[pos:]``. + + Returns ``(matched_length, value)``; ``matched_length`` is 0 when + no entry matches. + """ + node = self.root + matched_len = 0 + matched_value = None + i = pos + n = len(text) + while i < n: + child = node.children.get(text[i]) + if child is None: + break + node = child + i += 1 + if node.value is not None: + matched_len = i - pos + matched_value = node.value + return matched_len, matched_value + + +# DictGroup + +class _GroupMatcher: + """ + Ordered collection of tries. + + Mirrors the multi-table behaviour of ``PrefixMatch::MatchPrefix`` in the + C++ implementation: tries are checked in declaration order; the **first** + trie that has any prefix match returns its longest match for that trie. + This gives phrase dictionaries priority over character dictionaries. + """ + + __slots__ = ('tries',) + + def __init__(self, tries): + self.tries = list(tries) + + def match(self, text: str, pos: int): + """ + Return ``(matched_length, value)`` using priority-ordered trie lookup. + Returns ``(0, None)`` when no trie has a match. + """ + for trie in self.tries: + length, value = trie.prefix_match(text, pos) + if length > 0: + return length, value + return 0, None + + +# Dictionary loading + +_trie_cache: dict = {} # (absolute_path, reversed: bool) -> _Trie + + +def _find_dict_txt(filename: str): + """ + Resolve a dictionary filename (typically ``Foo.ocd2``) to a + ``(txt_path, needs_reverse)`` tuple. + + Resolution: + 1. Strip the extension (``Foo.ocd2`` → ``Foo``). + 2. Look for ``.txt`` in the bundled pkg dir, then the repo dir. + 3. If the stem ends with ``Rev`` (e.g. ``HKVariantsRev``), look for + the forward dict (``HKVariants.txt``) and mark it for reversal. + These reversed dicts are generated on-the-fly from the forward file, + matching the behaviour of the ``reverse_items`` build script. + + Raises ``FileNotFoundError`` if no suitable txt file is found. + """ + stem = filename + for ext in ('.ocd2', '.ocd', '.txt'): + if stem.endswith(ext): + stem = stem[: -len(ext)] + break + + for search_dir in (_pkg_dict_dir, _repo_dict_dir): + if not search_dir: + continue + path = os.path.join(search_dir, stem + '.txt') + if os.path.isfile(path): + return path, False + + # Try reversed dict: strip the trailing "Rev" and reverse the forward file. + if stem.endswith('Rev'): + forward_stem = stem[:-3] + for search_dir in (_pkg_dict_dir, _repo_dict_dir): + if not search_dir: + continue + path = os.path.join(search_dir, forward_stem + '.txt') + if os.path.isfile(path): + return path, True + + raise FileNotFoundError(f'Dictionary file not found for: {filename!r}') + + +def _parse_dict_lines(path: str): + """ + Yield ``(key, candidates)`` tuples for every data line in a txt dict file. + + Comment lines (starting with ``#``) and blank lines are skipped. + ``candidates`` is a list of one or more whitespace-separated values. + """ + with open(path, encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#') or '\t' not in line: + continue + key, rest = line.split('\t', 1) + candidates = rest.split() + if candidates: + yield key, candidates + + +def _load_trie(path: str, reverse: bool) -> _Trie: + """ + Load a ``keyvalue1 value2 ...`` txt file into a :class:`_Trie`. + + When ``reverse`` is ``True`` the mapping is inverted: each candidate + value becomes a key whose conversion target is the original key. When + multiple original keys share the same candidate value the first + encountered (in file order) wins, matching the behaviour of the C++ + dict-reversal script. + """ + cache_key = (path, reverse) + cached = _trie_cache.get(cache_key) + if cached is not None: + return cached + + trie = _Trie() + + if not reverse: + for key, candidates in _parse_dict_lines(path): + trie.add(key, candidates[0]) + else: + # Reverse: each candidate value -> original key (first writer wins). + reversed_mapping: dict = {} + for key, candidates in _parse_dict_lines(path): + for candidate in candidates: + if candidate not in reversed_mapping: + reversed_mapping[candidate] = key + for new_key, new_value in reversed_mapping.items(): + trie.add(new_key, new_value) + + _trie_cache[cache_key] = trie + return trie + + +def _get_trie(filename: str) -> _Trie: + """Resolve ``filename`` to a txt path and return the loaded trie.""" + path, needs_reverse = _find_dict_txt(filename) + return _load_trie(path, needs_reverse) + + +# Config parsing + +def _parse_dict_node(d: dict) -> _GroupMatcher: + """ + Recursively parse a JSON dict-config node and return a :class:`_GroupMatcher`. + + Mirrors ``ConfigInternal::ParseDict`` in ``Config.cpp``. Groups are + flattened into an ordered list of leaf tries, preserving the priority + order used by ``PrefixMatch::AddDict``. + """ + dtype = d['type'] + + if dtype == 'group': + tries = [] + for sub in d['dicts']: + sub_matcher = _parse_dict_node(sub) + # Flatten: extend with the sub-group's ordered tries list. + tries.extend(sub_matcher.tries) + return _GroupMatcher(tries) + + if dtype in ('ocd2', 'ocd', 'txt', 'text'): + trie = _get_trie(d['file']) + return _GroupMatcher([trie]) + + raise ValueError(f'Unknown dict type: {dtype!r}') + + +# Core conversion routines + +def _convert_phrase(text: str, matcher: _GroupMatcher) -> str: + """ + Convert *text* using left-to-right longest-prefix matching. + + At each position the group matcher is consulted (first-trie-wins + semantics). On a match the matched substring is replaced by the + dictionary value. On no match exactly one character is passed through + unchanged, mirroring ``Conversion::AppendConverted`` in the C++ code. + """ + parts = [] + i = 0 + n = len(text) + while i < n: + length, value = matcher.match(text, i) + if length > 0: + parts.append(value) + i += length + else: + parts.append(text[i]) + i += 1 + return ''.join(parts) + + +def _segment(text: str, matcher: _GroupMatcher) -> list: + """ + Max-match segmentation of *text*. + + Scans left-to-right using the segmentation dict. On a match the + pending character buffer is flushed as one segment and the matched key + becomes the next segment. Unmatched characters accumulate until the + next match or end of string. + + Mirrors ``MaxMatchSegmentation::Segment`` in the C++ code. + """ + segments = [] + buf = [] + i = 0 + n = len(text) + while i < n: + length, _ = matcher.match(text, i) + if length > 0: + if buf: + segments.append(''.join(buf)) + buf = [] + segments.append(text[i: i + length]) + i += length + else: + buf.append(text[i]) + i += 1 + if buf: + segments.append(''.join(buf)) + return segments + + +# Config location helpers + +def _find_config(config_name: str) -> str: + """ + Locate a config JSON file by name (with or without ``.json`` suffix). + + Searches the bundled package config directory first, then the source-tree + ``data/config`` directory as a fallback. + """ + filename = config_name if config_name.endswith('.json') else config_name + '.json' + for search_dir in (_pkg_config_dir, _repo_config_dir): + if not search_dir: + continue + path = os.path.join(search_dir, filename) + if os.path.isfile(path): + return path + raise FileNotFoundError(f'Config not found: {config_name!r}') + + +def _is_mmseg_config(config_path: str) -> bool: + """Return True if the config uses mmseg segmentation (pure-Python compatible).""" + try: + with open(config_path, encoding='utf-8') as f: + cfg = json.load(f) + return cfg.get('segmentation', {}).get('type') == 'mmseg' + except Exception: + return False + + +def list_configs() -> list: + """ + Return a sorted list of ``.json`` config filenames available to the + pure Python converter (mmseg-based configs only). + """ + configs = [] + seen = set() + for search_dir in (_pkg_config_dir, _repo_config_dir): + if not search_dir or not os.path.isdir(search_dir): + continue + for fname in os.listdir(search_dir): + if not fname.endswith('.json') or fname in seen: + continue + path = os.path.join(search_dir, fname) + if _is_mmseg_config(path): + configs.append(fname) + seen.add(fname) + return sorted(configs) + + +# Public API + +class OpenCC: + """ + Pure Python OpenCC converter. + + Usage:: + + cc = OpenCC('s2t') # or 's2t.json' + result = cc.convert('汉字') # -> '漢字' + + The ``config`` argument accepts a conversion name (e.g. ``'s2t'``) or + a bare filename (e.g. ``'s2t.json'``). Custom absolute config paths + are also accepted. + """ + + def __init__(self, config: str = 't2s') -> None: + config_name = config[:-5] if config.endswith('.json') else config + + # Allow custom config paths, matching the previous Python wrapper. + if os.path.isfile(config): + config_path = config + else: + config_path = _find_config(config_name) + + with open(config_path, encoding='utf-8') as f: + cfg = json.load(f) + + self.config = config if config.endswith('.json') else f'{config}.json' + + seg_type = cfg.get('segmentation', {}).get('type', '') + if seg_type != 'mmseg': + raise ValueError( + f"Config {config_name!r} uses segmentation type {seg_type!r} " + f"which is not supported by the pure Python backend." + ) + + self._segmenter: _GroupMatcher = _parse_dict_node( + cfg['segmentation']['dict'] + ) + self._chain: list = [ + _parse_dict_node(step['dict']) + for step in cfg['conversion_chain'] + ] + + def convert(self, text: str) -> str: + """Convert *text* and return the result.""" + segments = _segment(text, self._segmenter) + result = [] + for seg in segments: + converted = seg + for matcher in self._chain: + converted = _convert_phrase(converted, matcher) + result.append(converted) + return ''.join(result) diff --git a/python/tests/test_opencc.py b/python/tests/test_opencc.py index 6507fdab2..ddfd294ac 100644 --- a/python/tests/test_opencc.py +++ b/python/tests/test_opencc.py @@ -64,6 +64,12 @@ def test_init_delete_converter(): del converter +def test_jieba_configs_are_not_exposed(): + import opencc + + assert all('jieba' not in config for config in opencc.CONFIGS) + + def test_conversion(): import opencc import re diff --git a/release-pypi-linux.sh b/release-pypi-linux.sh index 477ea787a..23feb8a05 100644 --- a/release-pypi-linux.sh +++ b/release-pypi-linux.sh @@ -1,23 +1,11 @@ #!/bin/sh set -e -# Linux wheels are built inside manylinux2014 containers managed by cibuildwheel. -python -m pip install --upgrade pip cibuildwheel twine build +python -m pip install --upgrade pip build pytest twine -export CIBW_BUILD="cp310-* cp311-* cp312-* cp313-* cp314-*" -export CIBW_SKIP="*-musllinux_*" -export CIBW_MANYLINUX_X86_64_IMAGE="manylinux2014" -export CIBW_MANYLINUX_AARCH64_IMAGE="manylinux2014" -export CIBW_ENVIRONMENT="VERSION=\"${VERSION}\"" -export CIBW_TEST_REQUIRES="pytest" -export CIBW_TEST_COMMAND="pytest {project}/python/tests" - -python -m cibuildwheel --output-dir wheelhouse - -if [ "$(uname -m)" = "x86_64" ]; then - # Build sdist only once, to avoid duplicate uploads from matrix jobs. - python -m build --sdist --outdir wheelhouse -fi +python -m build --sdist --wheel --outdir wheelhouse +python -m pip install wheelhouse/*.whl +pytest python/tests if [ "$1" != "testonly" ]; then # Upload to PyPI diff --git a/release-pypi-macos.sh b/release-pypi-macos.sh index d1b95a9dc..0edeaf079 100644 --- a/release-pypi-macos.sh +++ b/release-pypi-macos.sh @@ -1,14 +1,11 @@ #!/bin/sh set -e -python -m pip install --upgrade pip cibuildwheel twine +python -m pip install --upgrade pip build pytest twine -export CIBW_BUILD="cp310-* cp311-* cp312-* cp313-* cp314-*" -export CIBW_ENVIRONMENT="VERSION=\"${VERSION}\"" -export CIBW_TEST_REQUIRES="pytest" -export CIBW_TEST_COMMAND="pytest {project}/python/tests" - -python -m cibuildwheel --output-dir wheelhouse +python -m build --wheel --outdir wheelhouse +python -m pip install wheelhouse/*.whl +pytest python/tests if [ "$1" != "testonly" ]; then # Upload to PyPI diff --git a/release-pypi-windows.cmd b/release-pypi-windows.cmd index f2a80874b..092822059 100644 --- a/release-pypi-windows.cmd +++ b/release-pypi-windows.cmd @@ -1,16 +1,18 @@ @echo off setlocal EnableDelayedExpansion -python -m pip install --upgrade pip cibuildwheel twine +python -m pip install --upgrade pip build pytest twine if !ERRORLEVEL! NEQ 0 (EXIT !ERRORLEVEL!) -SET CIBW_BUILD=cp310-* cp311-* cp312-* cp313-* cp314-* -SET CIBW_ARCHS=AMD64 -SET CIBW_ENVIRONMENT=VERSION="%VERSION%" -SET CIBW_TEST_REQUIRES=pytest -SET CIBW_TEST_COMMAND=pytest {project}/python/tests +python -m build --wheel --outdir wheelhouse +if !ERRORLEVEL! NEQ 0 (EXIT !ERRORLEVEL!) + +for %%f in (wheelhouse\*.whl) do ( + python -m pip install "%%f" + if !ERRORLEVEL! NEQ 0 (EXIT !ERRORLEVEL!) +) -python -m cibuildwheel --output-dir wheelhouse +pytest python/tests if !ERRORLEVEL! NEQ 0 (EXIT !ERRORLEVEL!) if NOT "%~1"=="testonly" ( From 971a1ca56ae2e4b625fa411545e382cdad8a6a78 Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Thu, 7 May 2026 08:36:20 -0700 Subject: [PATCH 02/13] Add Python golden tests and benchmarks, and update README.md --- README.md | 7 + python/benchmarks/benchmark_opencc.py | 189 ++++++++++++++++++++++++++ python/opencc/BUILD.bazel | 3 +- python/opencc/_opencc_pure.py | 55 ++++++-- python/tests/BUILD.bazel | 18 +++ python/tests/test_opencc.py | 10 ++ python/tests/test_opencc_golden.py | 59 ++++++++ 7 files changed, 326 insertions(+), 15 deletions(-) create mode 100644 python/benchmarks/benchmark_opencc.py create mode 100644 python/tests/test_opencc_golden.py diff --git a/README.md b/README.md index 92d13f402..2c53fee52 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,13 @@ The Python package uses the pure Python implementation and supports the built-in mmseg-based configs. Jieba plugin configs are not supported by the Python package. +The pure Python implementation loads text dictionaries. Built-in configs keep +their standard `.ocd2` dictionary references, and the Python loader resolves +those references to bundled same-stem `.txt` dictionaries when possible. Custom +configs that reference `.ocd` or `.ocd2` dictionaries must also provide matching +text dictionary files; binary dictionary files are not parsed by the Python +package. + ```python import opencc converter = opencc.OpenCC('s2t.json') diff --git a/python/benchmarks/benchmark_opencc.py b/python/benchmarks/benchmark_opencc.py new file mode 100644 index 000000000..16067e14b --- /dev/null +++ b/python/benchmarks/benchmark_opencc.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Benchmark the pure Python OpenCC package with project data.""" + +import argparse +import json +import statistics +import sys +import time +from pathlib import Path + +import opencc +from opencc import _opencc_pure + +_this_dir = Path(__file__).resolve().parent +_opencc_rootdir = _this_dir.parents[1] +_testcases_path = _opencc_rootdir / 'test' / 'testcases' / 'testcases.json' +_golden_input_path = _opencc_rootdir / 'test' / 'golden' / 'input' / 'us_constitution_zhs.txt' +_long_input_path = _opencc_rootdir / 'test' / 'benchmark' / 'zuozhuan.txt' + + +def _config_name(config): + return config[:-5] if config.endswith('.json') else config + + +def _default_configs(): + preferred = ['s2t', 't2s', 's2tw', 's2twp', 's2hk', 'tw2s'] + available = { + _config_name(config) + for config in opencc.CONFIGS + if 'jieba' not in config + } + return [config for config in preferred if config in available] + + +def _load_texts(dataset): + if dataset == 'testcases': + with open(_testcases_path, encoding='utf-8') as f: + parsed = json.load(f) + return [ + case['input'] + for case in parsed.get('cases', []) + if case.get('input') + ] + + if dataset == 'golden': + return [_golden_input_path.read_text(encoding='utf-8')] + + if dataset == 'long': + return [_long_input_path.read_text(encoding='utf-8')] + + raise ValueError(f'Unknown dataset: {dataset}') + + +def _time_call(fn): + start = time.perf_counter() + result = fn() + return time.perf_counter() - start, result + + +def _median(values): + return statistics.median(values) if values else 0.0 + + +def _benchmark_init(configs, iterations): + rows = [] + for config in configs: + samples = [] + for _ in range(iterations): + _opencc_pure._trie_cache.clear() + elapsed, _ = _time_call(lambda: opencc.OpenCC(config)) + samples.append(elapsed) + rows.append({ + 'name': f'init/{config}', + 'iterations': iterations, + 'seconds_median': _median(samples), + 'seconds_min': min(samples), + 'seconds_max': max(samples), + 'chars_per_second': None, + }) + return rows + + +def _benchmark_convert(configs, texts, warmups, iterations): + total_chars = sum(len(text) for text in texts) + rows = [] + for config in configs: + converter = opencc.OpenCC(config) + for _ in range(warmups): + for text in texts: + converter.convert(text) + + samples = [] + for _ in range(iterations): + elapsed, _ = _time_call( + lambda: [converter.convert(text) for text in texts] + ) + samples.append(elapsed) + + median = _median(samples) + rows.append({ + 'name': f'convert/{config}', + 'iterations': iterations, + 'seconds_median': median, + 'seconds_min': min(samples), + 'seconds_max': max(samples), + 'chars_per_second': total_chars / median if median > 0 else 0.0, + }) + return rows + + +def _print_table(rows): + print('| benchmark | iterations | median ms | min ms | max ms | chars/s |') + print('|---|---:|---:|---:|---:|---:|') + for row in rows: + chars_per_second = row['chars_per_second'] + if chars_per_second is None: + throughput = '' + else: + throughput = f'{chars_per_second:,.0f}' + print( + '| {name} | {iterations} | {median:.3f} | {min:.3f} | {max:.3f} | {throughput} |'.format( + name=row['name'], + iterations=row['iterations'], + median=row['seconds_median'] * 1000, + min=row['seconds_min'] * 1000, + max=row['seconds_max'] * 1000, + throughput=throughput, + ) + ) + + +def _print_json(rows): + print(json.dumps(rows, ensure_ascii=False, indent=2)) + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument( + '--dataset', + choices=['testcases', 'golden', 'long'], + default='golden', + help='Project dataset to use for conversion benchmarks.', + ) + parser.add_argument( + '--configs', + nargs='+', + default=None, + help='Config names to benchmark. Defaults to common non-jieba configs.', + ) + parser.add_argument( + '--warmups', + type=int, + default=1, + help='Warmup conversion runs before timing.', + ) + parser.add_argument( + '--iterations', + type=int, + default=5, + help='Timed iterations per benchmark.', + ) + parser.add_argument( + '--include-init', + action='store_true', + help='Also benchmark converter construction time.', + ) + parser.add_argument( + '--json', + action='store_true', + help='Emit machine-readable JSON instead of a Markdown table.', + ) + args = parser.parse_args(argv) + + configs = [_config_name(config) for config in (args.configs or _default_configs())] + texts = _load_texts(args.dataset) + + rows = [] + if args.include_init: + rows.extend(_benchmark_init(configs, args.iterations)) + rows.extend(_benchmark_convert(configs, texts, args.warmups, args.iterations)) + + if args.json: + _print_json(rows) + else: + _print_table(rows) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/python/opencc/BUILD.bazel b/python/opencc/BUILD.bazel index 227dd31e5..18501ec9f 100644 --- a/python/opencc/BUILD.bazel +++ b/python/opencc/BUILD.bazel @@ -6,16 +6,15 @@ py_library( name = "opencc", srcs = [ "__init__.py", + "_opencc_pure.py", "cli.py", ], data = [ "//data/config", - "//data/dictionary:binary_dictionaries", "//data/dictionary:text_dictionaries", ] + select({ "@platforms//os:windows": ["//python/opencc/clib:opencc_clib_pyd"], "//conditions:default": [], }), imports = [".."], - deps = ["//src:py_opencc"], ) diff --git a/python/opencc/_opencc_pure.py b/python/opencc/_opencc_pure.py index 4647fae95..59ac25ee5 100644 --- a/python/opencc/_opencc_pure.py +++ b/python/opencc/_opencc_pure.py @@ -38,6 +38,7 @@ import json import os +from pathlib import Path __all__ = ['OpenCC'] @@ -52,6 +53,20 @@ _MAX_PARENT_TRAVERSAL_DEPTH = 8 +def _runfile_dir(relative_path: str) -> str: + runfiles_root = os.environ.get('RUNFILES_DIR') + workspace = os.environ.get('TEST_WORKSPACE', '_main') + if not runfiles_root: + return '' + + for candidate in ( + Path(runfiles_root) / workspace / relative_path, + Path(runfiles_root) / relative_path): + if candidate.is_dir(): + return str(candidate) + return '' + + def _find_repo_root() -> str: """ Walk up the directory tree to find the OpenCC repo root. @@ -76,6 +91,8 @@ def _find_repo_root() -> str: _repo_root = _find_repo_root() _repo_config_dir = os.path.join(_repo_root, 'data', 'config') if _repo_root else '' _repo_dict_dir = os.path.join(_repo_root, 'data', 'dictionary') if _repo_root else '' +_runfiles_config_dir = _runfile_dir('data/config') +_runfiles_dict_dir = _runfile_dir('data/dictionary') # Trie @@ -171,14 +188,15 @@ def match(self, text: str, pos: int): _trie_cache: dict = {} # (absolute_path, reversed: bool) -> _Trie -def _find_dict_txt(filename: str): +def _find_dict_txt(filename: str, config_dir: str = ''): """ Resolve a dictionary filename (typically ``Foo.ocd2``) to a ``(txt_path, needs_reverse)`` tuple. Resolution: 1. Strip the extension (``Foo.ocd2`` → ``Foo``). - 2. Look for ``.txt`` in the bundled pkg dir, then the repo dir. + 2. Look for ``.txt`` next to a custom config, in the bundled pkg + dir, then in Bazel runfiles and the repo dir. 3. If the stem ends with ``Rev`` (e.g. ``HKVariantsRev``), look for the forward dict (``HKVariants.txt``) and mark it for reversal. These reversed dicts are generated on-the-fly from the forward file, @@ -192,7 +210,17 @@ def _find_dict_txt(filename: str): stem = stem[: -len(ext)] break - for search_dir in (_pkg_dict_dir, _repo_dict_dir): + if os.path.isabs(filename) and os.path.isfile(filename): + return filename, False + + search_dirs = ( + config_dir, + _pkg_dict_dir, + _runfiles_dict_dir, + _repo_dict_dir, + ) + + for search_dir in search_dirs: if not search_dir: continue path = os.path.join(search_dir, stem + '.txt') @@ -202,7 +230,7 @@ def _find_dict_txt(filename: str): # Try reversed dict: strip the trailing "Rev" and reverse the forward file. if stem.endswith('Rev'): forward_stem = stem[:-3] - for search_dir in (_pkg_dict_dir, _repo_dict_dir): + for search_dir in search_dirs: if not search_dir: continue path = os.path.join(search_dir, forward_stem + '.txt') @@ -264,15 +292,15 @@ def _load_trie(path: str, reverse: bool) -> _Trie: return trie -def _get_trie(filename: str) -> _Trie: +def _get_trie(filename: str, config_dir: str = '') -> _Trie: """Resolve ``filename`` to a txt path and return the loaded trie.""" - path, needs_reverse = _find_dict_txt(filename) + path, needs_reverse = _find_dict_txt(filename, config_dir) return _load_trie(path, needs_reverse) # Config parsing -def _parse_dict_node(d: dict) -> _GroupMatcher: +def _parse_dict_node(d: dict, config_dir: str = '') -> _GroupMatcher: """ Recursively parse a JSON dict-config node and return a :class:`_GroupMatcher`. @@ -285,13 +313,13 @@ def _parse_dict_node(d: dict) -> _GroupMatcher: if dtype == 'group': tries = [] for sub in d['dicts']: - sub_matcher = _parse_dict_node(sub) + sub_matcher = _parse_dict_node(sub, config_dir) # Flatten: extend with the sub-group's ordered tries list. tries.extend(sub_matcher.tries) return _GroupMatcher(tries) if dtype in ('ocd2', 'ocd', 'txt', 'text'): - trie = _get_trie(d['file']) + trie = _get_trie(d['file'], config_dir) return _GroupMatcher([trie]) raise ValueError(f'Unknown dict type: {dtype!r}') @@ -363,7 +391,7 @@ def _find_config(config_name: str) -> str: ``data/config`` directory as a fallback. """ filename = config_name if config_name.endswith('.json') else config_name + '.json' - for search_dir in (_pkg_config_dir, _repo_config_dir): + for search_dir in (_pkg_config_dir, _runfiles_config_dir, _repo_config_dir): if not search_dir: continue path = os.path.join(search_dir, filename) @@ -389,7 +417,7 @@ def list_configs() -> list: """ configs = [] seen = set() - for search_dir in (_pkg_config_dir, _repo_config_dir): + for search_dir in (_pkg_config_dir, _runfiles_config_dir, _repo_config_dir): if not search_dir or not os.path.isdir(search_dir): continue for fname in os.listdir(search_dir): @@ -440,10 +468,11 @@ def __init__(self, config: str = 't2s') -> None: ) self._segmenter: _GroupMatcher = _parse_dict_node( - cfg['segmentation']['dict'] + cfg['segmentation']['dict'], + os.path.dirname(config_path), ) self._chain: list = [ - _parse_dict_node(step['dict']) + _parse_dict_node(step['dict'], os.path.dirname(config_path)) for step in cfg['conversion_chain'] ] diff --git a/python/tests/BUILD.bazel b/python/tests/BUILD.bazel index b04da8956..7db99b53c 100644 --- a/python/tests/BUILD.bazel +++ b/python/tests/BUILD.bazel @@ -6,6 +6,7 @@ py_test( size = "small", srcs = ["test_opencc.py"], data = [ + "//test/config_test", "//test/testcases", ], imports = [".."], @@ -29,3 +30,20 @@ py_test( requirement("exceptiongroup"), ], ) + +py_test( + name = "test_opencc_golden", + size = "medium", + srcs = ["test_opencc_golden.py"], + data = [ + "//test/golden:golden_inputs", + "//test/golden:golden_outputs", + ], + imports = [".."], + deps = [ + "//python/opencc", + requirement("pytest"), + requirement("pygments"), + requirement("exceptiongroup"), + ], +) diff --git a/python/tests/test_opencc.py b/python/tests/test_opencc.py index ddfd294ac..c41e7c015 100644 --- a/python/tests/test_opencc.py +++ b/python/tests/test_opencc.py @@ -7,6 +7,7 @@ _this_dir = os.path.dirname(os.path.abspath(__file__)) _opencc_rootdir = os.path.abspath(os.path.join(_this_dir, '..', '..')) _testcases_path = os.path.join(_opencc_rootdir, 'test', 'testcases', 'testcases.json') +_config_test_path = os.path.join(_opencc_rootdir, 'test', 'config_test', 'config_test.json') _TEXT_ZIP_CONFIG = """{ "name": "Python Resource Zip Test", @@ -70,6 +71,15 @@ def test_jieba_configs_are_not_exposed(): assert all('jieba' not in config for config in opencc.CONFIGS) +def test_custom_config_resolves_local_dictionaries(): + import opencc + + converter = opencc.OpenCC(_config_test_path) + + assert converter.convert('燕燕于飞，之子于归。') == '燕燕于飛，之子于歸。' + assert converter.convert('远') == '遠' + + def test_conversion(): import opencc import re diff --git a/python/tests/test_opencc_golden.py b/python/tests/test_opencc_golden.py new file mode 100644 index 000000000..71c50b194 --- /dev/null +++ b/python/tests/test_opencc_golden.py @@ -0,0 +1,59 @@ +import difflib +from pathlib import Path + +import pytest + +import opencc + +_this_dir = Path(__file__).resolve().parent +_opencc_rootdir = _this_dir.parents[1] +_golden_input_dir = _opencc_rootdir / 'test' / 'golden' / 'input' +_golden_output_dir = _opencc_rootdir / 'test' / 'golden' / 'output' + +_GOLDEN_INPUTS = [ + 'us_constitution_zhs.txt', +] + + +def _available_python_golden_cases(): + configs = [ + config[:-5] if config.endswith('.json') else config + for config in opencc.CONFIGS + if 'jieba' not in config + ] + cases = [] + for input_name in _GOLDEN_INPUTS: + input_stem = Path(input_name).stem + for config in configs: + expected_path = _golden_output_dir / f'{input_stem}.{config}.txt' + if expected_path.is_file(): + cases.append((input_name, config, expected_path)) + return cases + + +_GOLDEN_CASES = _available_python_golden_cases() + + +def test_python_golden_cases_are_available(): + assert _GOLDEN_CASES + + +@pytest.mark.parametrize( + 'input_name,config,expected_path', + _GOLDEN_CASES, +) +def test_python_matches_project_golden_output(input_name, config, expected_path): + input_path = _golden_input_dir / input_name + source_text = input_path.read_text(encoding='utf-8') + expected = expected_path.read_text(encoding='utf-8') + + actual = opencc.OpenCC(config).convert(source_text) + + assert actual == expected, ''.join( + difflib.unified_diff( + expected.splitlines(keepends=True), + actual.splitlines(keepends=True), + fromfile=str(expected_path), + tofile=f'opencc.OpenCC({config!r})', + ) + ) From 9ce49ae88cd1ef12f5df9e9310574e07d184e1df Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Thu, 7 May 2026 11:28:19 -0700 Subject: [PATCH 03/13] Fix Python dictionary path parsing --- python/opencc/_opencc_pure.py | 8 +++++--- python/tests/test_opencc.py | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/python/opencc/_opencc_pure.py b/python/opencc/_opencc_pure.py index 59ac25ee5..28269f6bf 100644 --- a/python/opencc/_opencc_pure.py +++ b/python/opencc/_opencc_pure.py @@ -210,8 +210,10 @@ def _find_dict_txt(filename: str, config_dir: str = ''): stem = stem[: -len(ext)] break - if os.path.isabs(filename) and os.path.isfile(filename): - return filename, False + if os.path.isabs(filename): + path = stem + '.txt' + if os.path.isfile(path): + return path, False search_dirs = ( config_dir, @@ -247,7 +249,7 @@ def _parse_dict_lines(path: str): Comment lines (starting with ``#``) and blank lines are skipped. ``candidates`` is a list of one or more whitespace-separated values. """ - with open(path, encoding='utf-8') as f: + with open(path, encoding='utf-8-sig') as f: for line in f: line = line.strip() if not line or line.startswith('#') or '\t' not in line: diff --git a/python/tests/test_opencc.py b/python/tests/test_opencc.py index c41e7c015..cfc309595 100644 --- a/python/tests/test_opencc.py +++ b/python/tests/test_opencc.py @@ -80,6 +80,44 @@ def test_custom_config_resolves_local_dictionaries(): assert converter.convert('远') == '遠' +def test_custom_config_resolves_absolute_ocd2_path_to_text_dictionary(tmp_path): + import opencc + + dict_txt = tmp_path / 'CustomPhrases.txt' + dict_txt.write_text('测试\t測試\n', encoding='utf-8') + config_path = tmp_path / 'custom.json' + config_path.write_text(json.dumps({ + 'name': 'Custom', + 'segmentation': { + 'type': 'mmseg', + 'dict': { + 'type': 'ocd2', + 'file': str(tmp_path / 'CustomPhrases.ocd2'), + }, + }, + 'conversion_chain': [{ + 'dict': { + 'type': 'ocd2', + 'file': str(tmp_path / 'CustomPhrases.ocd2'), + }, + }], + }), encoding='utf-8') + + converter = opencc.OpenCC(str(config_path)) + + assert converter.convert('测试') == '測試' + + +def test_dictionary_parser_strips_utf8_bom(tmp_path): + from opencc import _opencc_pure + + dict_txt = tmp_path / 'BomPhrases.txt' + dict_txt.write_text('\ufeff测试\t測試\n', encoding='utf-8') + trie = _opencc_pure._load_trie(str(dict_txt), reverse=False) + + assert trie.prefix_match('测试', 0) == (2, '測試') + + def test_conversion(): import opencc import re From 1c29e9efa4218848b52d02070edb8f6b7923ba8a Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Thu, 7 May 2026 21:09:37 -0700 Subject: [PATCH 04/13] Add optional Python jieba support --- python/opencc/BUILD.bazel | 3 + python/opencc/_opencc_pure.py | 174 +++++++++++++++++++++++++++++++--- python/tests/BUILD.bazel | 1 + python/tests/test_opencc.py | 46 ++++++++- 4 files changed, 210 insertions(+), 14 deletions(-) diff --git a/python/opencc/BUILD.bazel b/python/opencc/BUILD.bazel index 18501ec9f..5ec579b33 100644 --- a/python/opencc/BUILD.bazel +++ b/python/opencc/BUILD.bazel @@ -12,6 +12,9 @@ py_library( data = [ "//data/config", "//data/dictionary:text_dictionaries", + "//plugins/jieba:config", + "//plugins/jieba/deps/cppjieba:dict/jieba.dict.utf8", + "//plugins/jieba/deps/cppjieba:dict/user.dict.utf8", ] + select({ "@platforms//os:windows": ["//python/opencc/clib:opencc_clib_pyd"], "//conditions:default": [], diff --git a/python/opencc/_opencc_pure.py b/python/opencc/_opencc_pure.py index 28269f6bf..c0b0769ec 100644 --- a/python/opencc/_opencc_pure.py +++ b/python/opencc/_opencc_pure.py @@ -38,6 +38,7 @@ import json import os +import importlib.util from pathlib import Path __all__ = ['OpenCC'] @@ -48,6 +49,7 @@ # ``pip install .`` and the data files are included as package_data). _pkg_config_dir = os.path.join(_this_dir, 'config') _pkg_dict_dir = os.path.join(_this_dir, 'dictionary') +_pkg_jieba_dict_dir = os.path.join(_this_dir, 'jieba_dict') _MAX_PARENT_TRAVERSAL_DEPTH = 8 @@ -91,8 +93,12 @@ def _find_repo_root() -> str: _repo_root = _find_repo_root() _repo_config_dir = os.path.join(_repo_root, 'data', 'config') if _repo_root else '' _repo_dict_dir = os.path.join(_repo_root, 'data', 'dictionary') if _repo_root else '' +_repo_jieba_config_dir = os.path.join(_repo_root, 'plugins', 'jieba', 'data', 'config') if _repo_root else '' +_repo_jieba_dict_dir = os.path.join(_repo_root, 'plugins', 'jieba', 'deps', 'cppjieba', 'dict') if _repo_root else '' _runfiles_config_dir = _runfile_dir('data/config') _runfiles_dict_dir = _runfile_dir('data/dictionary') +_runfiles_jieba_config_dir = _runfile_dir('plugins/jieba/data/config') +_runfiles_jieba_dict_dir = _runfile_dir('plugins/jieba/deps/cppjieba/dict') # Trie @@ -183,6 +189,111 @@ def match(self, text: str, pos: int): return 0, None +# Jieba segmentation + +_jieba_tokenizer_cache: dict = {} + + +class _JiebaSegmenter: + """ + Optional Jieba segmenter backed by the ``jieba`` Python package. + + The dependency is intentionally lazy: importing ``opencc`` and using the + normal mmseg configs never imports or requires ``jieba``. + """ + + __slots__ = ('tokenizer',) + + def __init__(self, resources: dict, config_dir: str = ''): + try: + import jieba + except ImportError as exc: + raise ImportError( + "Config uses Jieba segmentation. Install the optional " + "dependency with `pip install jieba` to use *_jieba configs." + ) from exc + + dict_path = _find_jieba_resource( + resources.get('dict_path', ''), + config_dir, + 'jieba.dict.utf8', + ) + user_dict_path = _find_jieba_resource( + resources.get('user_dict_path', ''), + config_dir, + 'user.dict.utf8', + required=False, + ) + + cache_key = (dict_path, user_dict_path) + tokenizer = _jieba_tokenizer_cache.get(cache_key) + if tokenizer is None: + tokenizer = jieba.Tokenizer(dictionary=dict_path) + tokenizer.initialize() + if user_dict_path: + tokenizer.load_userdict(user_dict_path) + _jieba_tokenizer_cache[cache_key] = tokenizer + self.tokenizer = tokenizer + + def segment(self, text: str) -> list: + return list(self.tokenizer.cut(text, HMM=True)) + + +def _jieba_available() -> bool: + return importlib.util.find_spec('jieba') is not None + + +def _find_jieba_resource(raw_path: str, + config_dir: str, + fallback_name: str, + required: bool = True) -> str: + """ + Resolve Jieba resources for the pure Python optional backend. + + Plugin configs point at ``jieba_merged.ocd2`` for the C++ plugin. Python + ``jieba`` consumes cppjieba's text dictionaries directly, so that request + is mapped to ``jieba.dict.utf8`` plus ``user.dict.utf8``. + """ + candidates = [] + + if raw_path: + if raw_path.endswith('jieba_merged.ocd2'): + candidates.append(raw_path[:-len('jieba_merged.ocd2')] + fallback_name) + candidates.append(fallback_name) + else: + candidates.append(raw_path) + candidates.append(os.path.join(os.path.dirname(raw_path), fallback_name)) + candidates.append(fallback_name) + else: + candidates.append(fallback_name) + + search_dirs = ( + '', + config_dir, + os.path.dirname(config_dir) if config_dir else '', + _pkg_jieba_dict_dir, + _runfiles_jieba_dict_dir, + _repo_jieba_dict_dir, + ) + + seen = set() + for candidate in candidates: + for search_dir in search_dirs: + if not candidate: + continue + path = candidate if os.path.isabs(candidate) or not search_dir else os.path.join(search_dir, candidate) + norm = os.path.abspath(path) + if norm in seen: + continue + seen.add(norm) + if os.path.isfile(path): + return path + + if required: + raise FileNotFoundError(f'Jieba resource not found: {fallback_name!r}') + return '' + + # Dictionary loading _trie_cache: dict = {} # (absolute_path, reversed: bool) -> _Trie @@ -393,7 +504,12 @@ def _find_config(config_name: str) -> str: ``data/config`` directory as a fallback. """ filename = config_name if config_name.endswith('.json') else config_name + '.json' - for search_dir in (_pkg_config_dir, _runfiles_config_dir, _repo_config_dir): + for search_dir in ( + _pkg_config_dir, + _runfiles_config_dir, + _repo_config_dir, + _runfiles_jieba_config_dir, + _repo_jieba_config_dir): if not search_dir: continue path = os.path.join(search_dir, filename) @@ -412,21 +528,47 @@ def _is_mmseg_config(config_path: str) -> bool: return False +def _is_supported_config(config_path: str) -> bool: + """Return True if the pure Python backend can use this config now.""" + try: + with open(config_path, encoding='utf-8') as f: + cfg = json.load(f) + seg_type = cfg.get('segmentation', {}).get('type') + if seg_type == 'mmseg': + return True + if seg_type == 'jieba': + if not _jieba_available(): + return False + resources = cfg.get('segmentation', {}).get('resources', {}) + config_dir = os.path.dirname(config_path) + _find_jieba_resource(resources.get('dict_path', ''), config_dir, 'jieba.dict.utf8') + return True + return False + except Exception: + return False + + def list_configs() -> list: """ Return a sorted list of ``.json`` config filenames available to the - pure Python converter (mmseg-based configs only). + pure Python converter. Jieba configs are listed only when the optional + ``jieba`` package and dictionary data are available. """ configs = [] seen = set() - for search_dir in (_pkg_config_dir, _runfiles_config_dir, _repo_config_dir): + for search_dir in ( + _pkg_config_dir, + _runfiles_config_dir, + _repo_config_dir, + _runfiles_jieba_config_dir, + _repo_jieba_config_dir): if not search_dir or not os.path.isdir(search_dir): continue for fname in os.listdir(search_dir): if not fname.endswith('.json') or fname in seen: continue path = os.path.join(search_dir, fname) - if _is_mmseg_config(path): + if _is_supported_config(path): configs.append(fname) seen.add(fname) return sorted(configs) @@ -462,25 +604,33 @@ def __init__(self, config: str = 't2s') -> None: self.config = config if config.endswith('.json') else f'{config}.json' - seg_type = cfg.get('segmentation', {}).get('type', '') - if seg_type != 'mmseg': + config_dir = os.path.dirname(config_path) + segmentation = cfg.get('segmentation', {}) + seg_type = segmentation.get('type', '') + if seg_type == 'mmseg': + self._segmenter = _parse_dict_node(segmentation['dict'], config_dir) + elif seg_type == 'jieba': + self._segmenter = _JiebaSegmenter( + segmentation.get('resources', {}), + config_dir, + ) + else: raise ValueError( f"Config {config_name!r} uses segmentation type {seg_type!r} " f"which is not supported by the pure Python backend." ) - self._segmenter: _GroupMatcher = _parse_dict_node( - cfg['segmentation']['dict'], - os.path.dirname(config_path), - ) self._chain: list = [ - _parse_dict_node(step['dict'], os.path.dirname(config_path)) + _parse_dict_node(step['dict'], config_dir) for step in cfg['conversion_chain'] ] def convert(self, text: str) -> str: """Convert *text* and return the result.""" - segments = _segment(text, self._segmenter) + if isinstance(self._segmenter, _JiebaSegmenter): + segments = self._segmenter.segment(text) + else: + segments = _segment(text, self._segmenter) result = [] for seg in segments: converted = seg diff --git a/python/tests/BUILD.bazel b/python/tests/BUILD.bazel index 7db99b53c..d07abe519 100644 --- a/python/tests/BUILD.bazel +++ b/python/tests/BUILD.bazel @@ -6,6 +6,7 @@ py_test( size = "small", srcs = ["test_opencc.py"], data = [ + "//plugins/jieba:testcases", "//test/config_test", "//test/testcases", ], diff --git a/python/tests/test_opencc.py b/python/tests/test_opencc.py index cfc309595..e26adbc91 100644 --- a/python/tests/test_opencc.py +++ b/python/tests/test_opencc.py @@ -1,4 +1,5 @@ import json +import importlib.util import os import pytest import sys @@ -8,6 +9,18 @@ _opencc_rootdir = os.path.abspath(os.path.join(_this_dir, '..', '..')) _testcases_path = os.path.join(_opencc_rootdir, 'test', 'testcases', 'testcases.json') _config_test_path = os.path.join(_opencc_rootdir, 'test', 'config_test', 'config_test.json') +_jieba_testcases_path = os.path.join( + _opencc_rootdir, + 'plugins', + 'jieba', + 'tests', + 'data', + 'jieba_comparison_testcases.json', +) + + +def _has_jieba(): + return importlib.util.find_spec('jieba') is not None _TEXT_ZIP_CONFIG = """{ "name": "Python Resource Zip Test", @@ -65,10 +78,39 @@ def test_init_delete_converter(): del converter -def test_jieba_configs_are_not_exposed(): +def test_jieba_configs_are_exposed_only_when_optional_dependency_is_available(): + import opencc + + if _has_jieba(): + assert any('jieba' in config for config in opencc.CONFIGS) + else: + assert all('jieba' not in config for config in opencc.CONFIGS) + + +def test_jieba_config_requires_optional_dependency_when_missing(): + if _has_jieba(): + pytest.skip('jieba is installed') + + import opencc + + with pytest.raises(ImportError, match='pip install jieba'): + opencc.OpenCC('s2t_jieba') + + +@pytest.mark.skipif(not _has_jieba(), reason='jieba optional dependency is not installed') +def test_jieba_configs_match_plugin_comparison_cases(): import opencc - assert all('jieba' not in config for config in opencc.CONFIGS) + with open(_jieba_testcases_path, 'r', encoding='utf-8') as f: + parsed = json.load(f) + + for case in parsed['cases']: + for cfg, ans in case.get('expected', {}).items(): + if not cfg.endswith('_jieba'): + continue + converter = opencc.OpenCC(f'{cfg}.json') + assert converter.convert(case['input']) == ans, \ + 'Failed to convert {} for {} -> {}'.format(cfg, case['input'], ans) def test_custom_config_resolves_local_dictionaries(): From d9f13350ee719f50c15d785e0ef88cefbb699f9f Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Thu, 7 May 2026 21:44:32 -0700 Subject: [PATCH 05/13] Preserve suffixless Python config paths --- python/opencc/_opencc_pure.py | 3 +++ python/tests/test_opencc.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/python/opencc/_opencc_pure.py b/python/opencc/_opencc_pure.py index c0b0769ec..f99012b10 100644 --- a/python/opencc/_opencc_pure.py +++ b/python/opencc/_opencc_pure.py @@ -594,8 +594,11 @@ def __init__(self, config: str = 't2s') -> None: config_name = config[:-5] if config.endswith('.json') else config # Allow custom config paths, matching the previous Python wrapper. + suffixed_config = config if config.endswith('.json') else f'{config}.json' if os.path.isfile(config): config_path = config + elif os.path.isfile(suffixed_config): + config_path = suffixed_config else: config_path = _find_config(config_name) diff --git a/python/tests/test_opencc.py b/python/tests/test_opencc.py index e26adbc91..46e608c32 100644 --- a/python/tests/test_opencc.py +++ b/python/tests/test_opencc.py @@ -122,6 +122,14 @@ def test_custom_config_resolves_local_dictionaries(): assert converter.convert('远') == '遠' +def test_custom_config_path_accepts_suffixless_json_path(): + import opencc + + converter = opencc.OpenCC(_config_test_path[:-5]) + + assert converter.convert('燕燕于飞，之子于归。') == '燕燕于飛，之子于歸。' + + def test_custom_config_resolves_absolute_ocd2_path_to_text_dictionary(tmp_path): import opencc From 0d61bb9e0815a11c2c64447a8ea8565f23faf15c Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Sat, 27 Jun 2026 19:15:54 -0700 Subject: [PATCH 06/13] Restore python/ to match upstream/master baseline Remove files added by the pure Python PR (#29) commits during rebase (benchmark_opencc.py, _opencc_pure.py, test_opencc_golden.py) and restore the BUILD.bazel and test files to their opencc-wasm-develop state. The pure Python implementation now lives exclusively in python-pure/, not in the C-extension python/ package. --- python/benchmarks/benchmark_opencc.py | 189 -------- python/opencc/BUILD.bazel | 6 +- python/opencc/_opencc_pure.py | 643 -------------------------- python/tests/BUILD.bazel | 19 - python/tests/test_opencc.py | 104 ----- python/tests/test_opencc_golden.py | 59 --- 6 files changed, 2 insertions(+), 1018 deletions(-) delete mode 100644 python/benchmarks/benchmark_opencc.py delete mode 100644 python/opencc/_opencc_pure.py delete mode 100644 python/tests/test_opencc_golden.py diff --git a/python/benchmarks/benchmark_opencc.py b/python/benchmarks/benchmark_opencc.py deleted file mode 100644 index 16067e14b..000000000 --- a/python/benchmarks/benchmark_opencc.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python3 -"""Benchmark the pure Python OpenCC package with project data.""" - -import argparse -import json -import statistics -import sys -import time -from pathlib import Path - -import opencc -from opencc import _opencc_pure - -_this_dir = Path(__file__).resolve().parent -_opencc_rootdir = _this_dir.parents[1] -_testcases_path = _opencc_rootdir / 'test' / 'testcases' / 'testcases.json' -_golden_input_path = _opencc_rootdir / 'test' / 'golden' / 'input' / 'us_constitution_zhs.txt' -_long_input_path = _opencc_rootdir / 'test' / 'benchmark' / 'zuozhuan.txt' - - -def _config_name(config): - return config[:-5] if config.endswith('.json') else config - - -def _default_configs(): - preferred = ['s2t', 't2s', 's2tw', 's2twp', 's2hk', 'tw2s'] - available = { - _config_name(config) - for config in opencc.CONFIGS - if 'jieba' not in config - } - return [config for config in preferred if config in available] - - -def _load_texts(dataset): - if dataset == 'testcases': - with open(_testcases_path, encoding='utf-8') as f: - parsed = json.load(f) - return [ - case['input'] - for case in parsed.get('cases', []) - if case.get('input') - ] - - if dataset == 'golden': - return [_golden_input_path.read_text(encoding='utf-8')] - - if dataset == 'long': - return [_long_input_path.read_text(encoding='utf-8')] - - raise ValueError(f'Unknown dataset: {dataset}') - - -def _time_call(fn): - start = time.perf_counter() - result = fn() - return time.perf_counter() - start, result - - -def _median(values): - return statistics.median(values) if values else 0.0 - - -def _benchmark_init(configs, iterations): - rows = [] - for config in configs: - samples = [] - for _ in range(iterations): - _opencc_pure._trie_cache.clear() - elapsed, _ = _time_call(lambda: opencc.OpenCC(config)) - samples.append(elapsed) - rows.append({ - 'name': f'init/{config}', - 'iterations': iterations, - 'seconds_median': _median(samples), - 'seconds_min': min(samples), - 'seconds_max': max(samples), - 'chars_per_second': None, - }) - return rows - - -def _benchmark_convert(configs, texts, warmups, iterations): - total_chars = sum(len(text) for text in texts) - rows = [] - for config in configs: - converter = opencc.OpenCC(config) - for _ in range(warmups): - for text in texts: - converter.convert(text) - - samples = [] - for _ in range(iterations): - elapsed, _ = _time_call( - lambda: [converter.convert(text) for text in texts] - ) - samples.append(elapsed) - - median = _median(samples) - rows.append({ - 'name': f'convert/{config}', - 'iterations': iterations, - 'seconds_median': median, - 'seconds_min': min(samples), - 'seconds_max': max(samples), - 'chars_per_second': total_chars / median if median > 0 else 0.0, - }) - return rows - - -def _print_table(rows): - print('| benchmark | iterations | median ms | min ms | max ms | chars/s |') - print('|---|---:|---:|---:|---:|---:|') - for row in rows: - chars_per_second = row['chars_per_second'] - if chars_per_second is None: - throughput = '' - else: - throughput = f'{chars_per_second:,.0f}' - print( - '| {name} | {iterations} | {median:.3f} | {min:.3f} | {max:.3f} | {throughput} |'.format( - name=row['name'], - iterations=row['iterations'], - median=row['seconds_median'] * 1000, - min=row['seconds_min'] * 1000, - max=row['seconds_max'] * 1000, - throughput=throughput, - ) - ) - - -def _print_json(rows): - print(json.dumps(rows, ensure_ascii=False, indent=2)) - - -def main(argv=None): - parser = argparse.ArgumentParser() - parser.add_argument( - '--dataset', - choices=['testcases', 'golden', 'long'], - default='golden', - help='Project dataset to use for conversion benchmarks.', - ) - parser.add_argument( - '--configs', - nargs='+', - default=None, - help='Config names to benchmark. Defaults to common non-jieba configs.', - ) - parser.add_argument( - '--warmups', - type=int, - default=1, - help='Warmup conversion runs before timing.', - ) - parser.add_argument( - '--iterations', - type=int, - default=5, - help='Timed iterations per benchmark.', - ) - parser.add_argument( - '--include-init', - action='store_true', - help='Also benchmark converter construction time.', - ) - parser.add_argument( - '--json', - action='store_true', - help='Emit machine-readable JSON instead of a Markdown table.', - ) - args = parser.parse_args(argv) - - configs = [_config_name(config) for config in (args.configs or _default_configs())] - texts = _load_texts(args.dataset) - - rows = [] - if args.include_init: - rows.extend(_benchmark_init(configs, args.iterations)) - rows.extend(_benchmark_convert(configs, texts, args.warmups, args.iterations)) - - if args.json: - _print_json(rows) - else: - _print_table(rows) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/python/opencc/BUILD.bazel b/python/opencc/BUILD.bazel index 5ec579b33..227dd31e5 100644 --- a/python/opencc/BUILD.bazel +++ b/python/opencc/BUILD.bazel @@ -6,18 +6,16 @@ py_library( name = "opencc", srcs = [ "__init__.py", - "_opencc_pure.py", "cli.py", ], data = [ "//data/config", + "//data/dictionary:binary_dictionaries", "//data/dictionary:text_dictionaries", - "//plugins/jieba:config", - "//plugins/jieba/deps/cppjieba:dict/jieba.dict.utf8", - "//plugins/jieba/deps/cppjieba:dict/user.dict.utf8", ] + select({ "@platforms//os:windows": ["//python/opencc/clib:opencc_clib_pyd"], "//conditions:default": [], }), imports = [".."], + deps = ["//src:py_opencc"], ) diff --git a/python/opencc/_opencc_pure.py b/python/opencc/_opencc_pure.py deleted file mode 100644 index f99012b10..000000000 --- a/python/opencc/_opencc_pure.py +++ /dev/null @@ -1,643 +0,0 @@ -""" -Pure Python implementation of the OpenCC Chinese conversion algorithm. - -Algorithm overview (mirrors the C++ reference implementation): - - Dictionary (.txt) - Each line: ``keyvalue1 value2 ...`` - The first listed candidate is used as the conversion target. - - Trie - Character-by-character prefix tree. Built per dictionary file. - ``prefix_match(text, pos)`` returns ``(matched_length, value)`` - for the *longest* match starting at ``text[pos]``. - - DictGroup (_GroupMatcher) - An ordered list of tries. At each position the tries are tried in - declaration order; the **first** trie that has any match (not the - globally longest) wins. This gives phrase dictionaries priority - over character dictionaries, mirroring the C++ PrefixMatch logic. - - Conversion - Scans the text left-to-right. At each position the active - group/dict matcher is consulted; the matched substring is replaced - with the dictionary value. On no match, exactly one character is - passed through unchanged. - - MaxMatch Segmentation - Same scan as Conversion, but instead of replacing tokens the - matched keys are emitted as segment boundaries. Unmatched - characters accumulate in a buffer that is flushed as a single - segment when the next match is found or the string ends. - - Conversion Chain - Multiple conversion steps applied sequentially to each segment - produced by the segmenter. Each step uses an independent - group/dict matcher. -""" - -import json -import os -import importlib.util -from pathlib import Path - -__all__ = ['OpenCC'] - -_this_dir = os.path.dirname(os.path.abspath(__file__)) - -# Bundled data directories (populated when the package is installed with -# ``pip install .`` and the data files are included as package_data). -_pkg_config_dir = os.path.join(_this_dir, 'config') -_pkg_dict_dir = os.path.join(_this_dir, 'dictionary') -_pkg_jieba_dict_dir = os.path.join(_this_dir, 'jieba_dict') - - -_MAX_PARENT_TRAVERSAL_DEPTH = 8 - - -def _runfile_dir(relative_path: str) -> str: - runfiles_root = os.environ.get('RUNFILES_DIR') - workspace = os.environ.get('TEST_WORKSPACE', '_main') - if not runfiles_root: - return '' - - for candidate in ( - Path(runfiles_root) / workspace / relative_path, - Path(runfiles_root) / relative_path): - if candidate.is_dir(): - return str(candidate) - return '' - - -def _find_repo_root() -> str: - """ - Walk up the directory tree to find the OpenCC repo root. - - Detects the repo by the presence of ``data/config/`` and - ``data/dictionary/`` subdirectories. Returns an empty string when the - repo root cannot be determined (e.g. in a standalone installed package - without bundled data files). - """ - candidate = _this_dir - for _ in range(_MAX_PARENT_TRAVERSAL_DEPTH): - if (os.path.isdir(os.path.join(candidate, 'data', 'config')) and - os.path.isdir(os.path.join(candidate, 'data', 'dictionary'))): - return candidate - parent = os.path.dirname(candidate) - if parent == candidate: - break - candidate = parent - return '' - - -_repo_root = _find_repo_root() -_repo_config_dir = os.path.join(_repo_root, 'data', 'config') if _repo_root else '' -_repo_dict_dir = os.path.join(_repo_root, 'data', 'dictionary') if _repo_root else '' -_repo_jieba_config_dir = os.path.join(_repo_root, 'plugins', 'jieba', 'data', 'config') if _repo_root else '' -_repo_jieba_dict_dir = os.path.join(_repo_root, 'plugins', 'jieba', 'deps', 'cppjieba', 'dict') if _repo_root else '' -_runfiles_config_dir = _runfile_dir('data/config') -_runfiles_dict_dir = _runfile_dir('data/dictionary') -_runfiles_jieba_config_dir = _runfile_dir('plugins/jieba/data/config') -_runfiles_jieba_dict_dir = _runfile_dir('plugins/jieba/deps/cppjieba/dict') - - -# Trie - -class _TrieNode: - """Single node in the character trie.""" - - __slots__ = ('children', 'value') - - def __init__(self): - self.children = {} # char -> _TrieNode - self.value = None # str or None; non-None marks a terminal node - - -class _Trie: - """ - Character-by-character trie for longest-prefix matching. - - Mirrors the inner ``Table`` class in ``PrefixMatch.cpp``. - """ - - __slots__ = ('root',) - - def __init__(self): - self.root = _TrieNode() - - def add(self, key: str, value: str) -> None: - """Insert ``key -> value`` into the trie.""" - node = self.root - for ch in key: - child = node.children.get(ch) - if child is None: - child = _TrieNode() - node.children[ch] = child - node = child - node.value = value - - def prefix_match(self, text: str, pos: int): - """ - Find the longest entry whose key is a prefix of ``text[pos:]``. - - Returns ``(matched_length, value)``; ``matched_length`` is 0 when - no entry matches. - """ - node = self.root - matched_len = 0 - matched_value = None - i = pos - n = len(text) - while i < n: - child = node.children.get(text[i]) - if child is None: - break - node = child - i += 1 - if node.value is not None: - matched_len = i - pos - matched_value = node.value - return matched_len, matched_value - - -# DictGroup - -class _GroupMatcher: - """ - Ordered collection of tries. - - Mirrors the multi-table behaviour of ``PrefixMatch::MatchPrefix`` in the - C++ implementation: tries are checked in declaration order; the **first** - trie that has any prefix match returns its longest match for that trie. - This gives phrase dictionaries priority over character dictionaries. - """ - - __slots__ = ('tries',) - - def __init__(self, tries): - self.tries = list(tries) - - def match(self, text: str, pos: int): - """ - Return ``(matched_length, value)`` using priority-ordered trie lookup. - Returns ``(0, None)`` when no trie has a match. - """ - for trie in self.tries: - length, value = trie.prefix_match(text, pos) - if length > 0: - return length, value - return 0, None - - -# Jieba segmentation - -_jieba_tokenizer_cache: dict = {} - - -class _JiebaSegmenter: - """ - Optional Jieba segmenter backed by the ``jieba`` Python package. - - The dependency is intentionally lazy: importing ``opencc`` and using the - normal mmseg configs never imports or requires ``jieba``. - """ - - __slots__ = ('tokenizer',) - - def __init__(self, resources: dict, config_dir: str = ''): - try: - import jieba - except ImportError as exc: - raise ImportError( - "Config uses Jieba segmentation. Install the optional " - "dependency with `pip install jieba` to use *_jieba configs." - ) from exc - - dict_path = _find_jieba_resource( - resources.get('dict_path', ''), - config_dir, - 'jieba.dict.utf8', - ) - user_dict_path = _find_jieba_resource( - resources.get('user_dict_path', ''), - config_dir, - 'user.dict.utf8', - required=False, - ) - - cache_key = (dict_path, user_dict_path) - tokenizer = _jieba_tokenizer_cache.get(cache_key) - if tokenizer is None: - tokenizer = jieba.Tokenizer(dictionary=dict_path) - tokenizer.initialize() - if user_dict_path: - tokenizer.load_userdict(user_dict_path) - _jieba_tokenizer_cache[cache_key] = tokenizer - self.tokenizer = tokenizer - - def segment(self, text: str) -> list: - return list(self.tokenizer.cut(text, HMM=True)) - - -def _jieba_available() -> bool: - return importlib.util.find_spec('jieba') is not None - - -def _find_jieba_resource(raw_path: str, - config_dir: str, - fallback_name: str, - required: bool = True) -> str: - """ - Resolve Jieba resources for the pure Python optional backend. - - Plugin configs point at ``jieba_merged.ocd2`` for the C++ plugin. Python - ``jieba`` consumes cppjieba's text dictionaries directly, so that request - is mapped to ``jieba.dict.utf8`` plus ``user.dict.utf8``. - """ - candidates = [] - - if raw_path: - if raw_path.endswith('jieba_merged.ocd2'): - candidates.append(raw_path[:-len('jieba_merged.ocd2')] + fallback_name) - candidates.append(fallback_name) - else: - candidates.append(raw_path) - candidates.append(os.path.join(os.path.dirname(raw_path), fallback_name)) - candidates.append(fallback_name) - else: - candidates.append(fallback_name) - - search_dirs = ( - '', - config_dir, - os.path.dirname(config_dir) if config_dir else '', - _pkg_jieba_dict_dir, - _runfiles_jieba_dict_dir, - _repo_jieba_dict_dir, - ) - - seen = set() - for candidate in candidates: - for search_dir in search_dirs: - if not candidate: - continue - path = candidate if os.path.isabs(candidate) or not search_dir else os.path.join(search_dir, candidate) - norm = os.path.abspath(path) - if norm in seen: - continue - seen.add(norm) - if os.path.isfile(path): - return path - - if required: - raise FileNotFoundError(f'Jieba resource not found: {fallback_name!r}') - return '' - - -# Dictionary loading - -_trie_cache: dict = {} # (absolute_path, reversed: bool) -> _Trie - - -def _find_dict_txt(filename: str, config_dir: str = ''): - """ - Resolve a dictionary filename (typically ``Foo.ocd2``) to a - ``(txt_path, needs_reverse)`` tuple. - - Resolution: - 1. Strip the extension (``Foo.ocd2`` → ``Foo``). - 2. Look for ``.txt`` next to a custom config, in the bundled pkg - dir, then in Bazel runfiles and the repo dir. - 3. If the stem ends with ``Rev`` (e.g. ``HKVariantsRev``), look for - the forward dict (``HKVariants.txt``) and mark it for reversal. - These reversed dicts are generated on-the-fly from the forward file, - matching the behaviour of the ``reverse_items`` build script. - - Raises ``FileNotFoundError`` if no suitable txt file is found. - """ - stem = filename - for ext in ('.ocd2', '.ocd', '.txt'): - if stem.endswith(ext): - stem = stem[: -len(ext)] - break - - if os.path.isabs(filename): - path = stem + '.txt' - if os.path.isfile(path): - return path, False - - search_dirs = ( - config_dir, - _pkg_dict_dir, - _runfiles_dict_dir, - _repo_dict_dir, - ) - - for search_dir in search_dirs: - if not search_dir: - continue - path = os.path.join(search_dir, stem + '.txt') - if os.path.isfile(path): - return path, False - - # Try reversed dict: strip the trailing "Rev" and reverse the forward file. - if stem.endswith('Rev'): - forward_stem = stem[:-3] - for search_dir in search_dirs: - if not search_dir: - continue - path = os.path.join(search_dir, forward_stem + '.txt') - if os.path.isfile(path): - return path, True - - raise FileNotFoundError(f'Dictionary file not found for: {filename!r}') - - -def _parse_dict_lines(path: str): - """ - Yield ``(key, candidates)`` tuples for every data line in a txt dict file. - - Comment lines (starting with ``#``) and blank lines are skipped. - ``candidates`` is a list of one or more whitespace-separated values. - """ - with open(path, encoding='utf-8-sig') as f: - for line in f: - line = line.strip() - if not line or line.startswith('#') or '\t' not in line: - continue - key, rest = line.split('\t', 1) - candidates = rest.split() - if candidates: - yield key, candidates - - -def _load_trie(path: str, reverse: bool) -> _Trie: - """ - Load a ``keyvalue1 value2 ...`` txt file into a :class:`_Trie`. - - When ``reverse`` is ``True`` the mapping is inverted: each candidate - value becomes a key whose conversion target is the original key. When - multiple original keys share the same candidate value the first - encountered (in file order) wins, matching the behaviour of the C++ - dict-reversal script. - """ - cache_key = (path, reverse) - cached = _trie_cache.get(cache_key) - if cached is not None: - return cached - - trie = _Trie() - - if not reverse: - for key, candidates in _parse_dict_lines(path): - trie.add(key, candidates[0]) - else: - # Reverse: each candidate value -> original key (first writer wins). - reversed_mapping: dict = {} - for key, candidates in _parse_dict_lines(path): - for candidate in candidates: - if candidate not in reversed_mapping: - reversed_mapping[candidate] = key - for new_key, new_value in reversed_mapping.items(): - trie.add(new_key, new_value) - - _trie_cache[cache_key] = trie - return trie - - -def _get_trie(filename: str, config_dir: str = '') -> _Trie: - """Resolve ``filename`` to a txt path and return the loaded trie.""" - path, needs_reverse = _find_dict_txt(filename, config_dir) - return _load_trie(path, needs_reverse) - - -# Config parsing - -def _parse_dict_node(d: dict, config_dir: str = '') -> _GroupMatcher: - """ - Recursively parse a JSON dict-config node and return a :class:`_GroupMatcher`. - - Mirrors ``ConfigInternal::ParseDict`` in ``Config.cpp``. Groups are - flattened into an ordered list of leaf tries, preserving the priority - order used by ``PrefixMatch::AddDict``. - """ - dtype = d['type'] - - if dtype == 'group': - tries = [] - for sub in d['dicts']: - sub_matcher = _parse_dict_node(sub, config_dir) - # Flatten: extend with the sub-group's ordered tries list. - tries.extend(sub_matcher.tries) - return _GroupMatcher(tries) - - if dtype in ('ocd2', 'ocd', 'txt', 'text'): - trie = _get_trie(d['file'], config_dir) - return _GroupMatcher([trie]) - - raise ValueError(f'Unknown dict type: {dtype!r}') - - -# Core conversion routines - -def _convert_phrase(text: str, matcher: _GroupMatcher) -> str: - """ - Convert *text* using left-to-right longest-prefix matching. - - At each position the group matcher is consulted (first-trie-wins - semantics). On a match the matched substring is replaced by the - dictionary value. On no match exactly one character is passed through - unchanged, mirroring ``Conversion::AppendConverted`` in the C++ code. - """ - parts = [] - i = 0 - n = len(text) - while i < n: - length, value = matcher.match(text, i) - if length > 0: - parts.append(value) - i += length - else: - parts.append(text[i]) - i += 1 - return ''.join(parts) - - -def _segment(text: str, matcher: _GroupMatcher) -> list: - """ - Max-match segmentation of *text*. - - Scans left-to-right using the segmentation dict. On a match the - pending character buffer is flushed as one segment and the matched key - becomes the next segment. Unmatched characters accumulate until the - next match or end of string. - - Mirrors ``MaxMatchSegmentation::Segment`` in the C++ code. - """ - segments = [] - buf = [] - i = 0 - n = len(text) - while i < n: - length, _ = matcher.match(text, i) - if length > 0: - if buf: - segments.append(''.join(buf)) - buf = [] - segments.append(text[i: i + length]) - i += length - else: - buf.append(text[i]) - i += 1 - if buf: - segments.append(''.join(buf)) - return segments - - -# Config location helpers - -def _find_config(config_name: str) -> str: - """ - Locate a config JSON file by name (with or without ``.json`` suffix). - - Searches the bundled package config directory first, then the source-tree - ``data/config`` directory as a fallback. - """ - filename = config_name if config_name.endswith('.json') else config_name + '.json' - for search_dir in ( - _pkg_config_dir, - _runfiles_config_dir, - _repo_config_dir, - _runfiles_jieba_config_dir, - _repo_jieba_config_dir): - if not search_dir: - continue - path = os.path.join(search_dir, filename) - if os.path.isfile(path): - return path - raise FileNotFoundError(f'Config not found: {config_name!r}') - - -def _is_mmseg_config(config_path: str) -> bool: - """Return True if the config uses mmseg segmentation (pure-Python compatible).""" - try: - with open(config_path, encoding='utf-8') as f: - cfg = json.load(f) - return cfg.get('segmentation', {}).get('type') == 'mmseg' - except Exception: - return False - - -def _is_supported_config(config_path: str) -> bool: - """Return True if the pure Python backend can use this config now.""" - try: - with open(config_path, encoding='utf-8') as f: - cfg = json.load(f) - seg_type = cfg.get('segmentation', {}).get('type') - if seg_type == 'mmseg': - return True - if seg_type == 'jieba': - if not _jieba_available(): - return False - resources = cfg.get('segmentation', {}).get('resources', {}) - config_dir = os.path.dirname(config_path) - _find_jieba_resource(resources.get('dict_path', ''), config_dir, 'jieba.dict.utf8') - return True - return False - except Exception: - return False - - -def list_configs() -> list: - """ - Return a sorted list of ``.json`` config filenames available to the - pure Python converter. Jieba configs are listed only when the optional - ``jieba`` package and dictionary data are available. - """ - configs = [] - seen = set() - for search_dir in ( - _pkg_config_dir, - _runfiles_config_dir, - _repo_config_dir, - _runfiles_jieba_config_dir, - _repo_jieba_config_dir): - if not search_dir or not os.path.isdir(search_dir): - continue - for fname in os.listdir(search_dir): - if not fname.endswith('.json') or fname in seen: - continue - path = os.path.join(search_dir, fname) - if _is_supported_config(path): - configs.append(fname) - seen.add(fname) - return sorted(configs) - - -# Public API - -class OpenCC: - """ - Pure Python OpenCC converter. - - Usage:: - - cc = OpenCC('s2t') # or 's2t.json' - result = cc.convert('汉字') # -> '漢字' - - The ``config`` argument accepts a conversion name (e.g. ``'s2t'``) or - a bare filename (e.g. ``'s2t.json'``). Custom absolute config paths - are also accepted. - """ - - def __init__(self, config: str = 't2s') -> None: - config_name = config[:-5] if config.endswith('.json') else config - - # Allow custom config paths, matching the previous Python wrapper. - suffixed_config = config if config.endswith('.json') else f'{config}.json' - if os.path.isfile(config): - config_path = config - elif os.path.isfile(suffixed_config): - config_path = suffixed_config - else: - config_path = _find_config(config_name) - - with open(config_path, encoding='utf-8') as f: - cfg = json.load(f) - - self.config = config if config.endswith('.json') else f'{config}.json' - - config_dir = os.path.dirname(config_path) - segmentation = cfg.get('segmentation', {}) - seg_type = segmentation.get('type', '') - if seg_type == 'mmseg': - self._segmenter = _parse_dict_node(segmentation['dict'], config_dir) - elif seg_type == 'jieba': - self._segmenter = _JiebaSegmenter( - segmentation.get('resources', {}), - config_dir, - ) - else: - raise ValueError( - f"Config {config_name!r} uses segmentation type {seg_type!r} " - f"which is not supported by the pure Python backend." - ) - - self._chain: list = [ - _parse_dict_node(step['dict'], config_dir) - for step in cfg['conversion_chain'] - ] - - def convert(self, text: str) -> str: - """Convert *text* and return the result.""" - if isinstance(self._segmenter, _JiebaSegmenter): - segments = self._segmenter.segment(text) - else: - segments = _segment(text, self._segmenter) - result = [] - for seg in segments: - converted = seg - for matcher in self._chain: - converted = _convert_phrase(converted, matcher) - result.append(converted) - return ''.join(result) diff --git a/python/tests/BUILD.bazel b/python/tests/BUILD.bazel index d07abe519..b04da8956 100644 --- a/python/tests/BUILD.bazel +++ b/python/tests/BUILD.bazel @@ -6,8 +6,6 @@ py_test( size = "small", srcs = ["test_opencc.py"], data = [ - "//plugins/jieba:testcases", - "//test/config_test", "//test/testcases", ], imports = [".."], @@ -31,20 +29,3 @@ py_test( requirement("exceptiongroup"), ], ) - -py_test( - name = "test_opencc_golden", - size = "medium", - srcs = ["test_opencc_golden.py"], - data = [ - "//test/golden:golden_inputs", - "//test/golden:golden_outputs", - ], - imports = [".."], - deps = [ - "//python/opencc", - requirement("pytest"), - requirement("pygments"), - requirement("exceptiongroup"), - ], -) diff --git a/python/tests/test_opencc.py b/python/tests/test_opencc.py index 46e608c32..6507fdab2 100644 --- a/python/tests/test_opencc.py +++ b/python/tests/test_opencc.py @@ -1,5 +1,4 @@ import json -import importlib.util import os import pytest import sys @@ -8,19 +7,6 @@ _this_dir = os.path.dirname(os.path.abspath(__file__)) _opencc_rootdir = os.path.abspath(os.path.join(_this_dir, '..', '..')) _testcases_path = os.path.join(_opencc_rootdir, 'test', 'testcases', 'testcases.json') -_config_test_path = os.path.join(_opencc_rootdir, 'test', 'config_test', 'config_test.json') -_jieba_testcases_path = os.path.join( - _opencc_rootdir, - 'plugins', - 'jieba', - 'tests', - 'data', - 'jieba_comparison_testcases.json', -) - - -def _has_jieba(): - return importlib.util.find_spec('jieba') is not None _TEXT_ZIP_CONFIG = """{ "name": "Python Resource Zip Test", @@ -78,96 +64,6 @@ def test_init_delete_converter(): del converter -def test_jieba_configs_are_exposed_only_when_optional_dependency_is_available(): - import opencc - - if _has_jieba(): - assert any('jieba' in config for config in opencc.CONFIGS) - else: - assert all('jieba' not in config for config in opencc.CONFIGS) - - -def test_jieba_config_requires_optional_dependency_when_missing(): - if _has_jieba(): - pytest.skip('jieba is installed') - - import opencc - - with pytest.raises(ImportError, match='pip install jieba'): - opencc.OpenCC('s2t_jieba') - - -@pytest.mark.skipif(not _has_jieba(), reason='jieba optional dependency is not installed') -def test_jieba_configs_match_plugin_comparison_cases(): - import opencc - - with open(_jieba_testcases_path, 'r', encoding='utf-8') as f: - parsed = json.load(f) - - for case in parsed['cases']: - for cfg, ans in case.get('expected', {}).items(): - if not cfg.endswith('_jieba'): - continue - converter = opencc.OpenCC(f'{cfg}.json') - assert converter.convert(case['input']) == ans, \ - 'Failed to convert {} for {} -> {}'.format(cfg, case['input'], ans) - - -def test_custom_config_resolves_local_dictionaries(): - import opencc - - converter = opencc.OpenCC(_config_test_path) - - assert converter.convert('燕燕于飞，之子于归。') == '燕燕于飛，之子于歸。' - assert converter.convert('远') == '遠' - - -def test_custom_config_path_accepts_suffixless_json_path(): - import opencc - - converter = opencc.OpenCC(_config_test_path[:-5]) - - assert converter.convert('燕燕于飞，之子于归。') == '燕燕于飛，之子于歸。' - - -def test_custom_config_resolves_absolute_ocd2_path_to_text_dictionary(tmp_path): - import opencc - - dict_txt = tmp_path / 'CustomPhrases.txt' - dict_txt.write_text('测试\t測試\n', encoding='utf-8') - config_path = tmp_path / 'custom.json' - config_path.write_text(json.dumps({ - 'name': 'Custom', - 'segmentation': { - 'type': 'mmseg', - 'dict': { - 'type': 'ocd2', - 'file': str(tmp_path / 'CustomPhrases.ocd2'), - }, - }, - 'conversion_chain': [{ - 'dict': { - 'type': 'ocd2', - 'file': str(tmp_path / 'CustomPhrases.ocd2'), - }, - }], - }), encoding='utf-8') - - converter = opencc.OpenCC(str(config_path)) - - assert converter.convert('测试') == '測試' - - -def test_dictionary_parser_strips_utf8_bom(tmp_path): - from opencc import _opencc_pure - - dict_txt = tmp_path / 'BomPhrases.txt' - dict_txt.write_text('\ufeff测试\t測試\n', encoding='utf-8') - trie = _opencc_pure._load_trie(str(dict_txt), reverse=False) - - assert trie.prefix_match('测试', 0) == (2, '測試') - - def test_conversion(): import opencc import re diff --git a/python/tests/test_opencc_golden.py b/python/tests/test_opencc_golden.py deleted file mode 100644 index 71c50b194..000000000 --- a/python/tests/test_opencc_golden.py +++ /dev/null @@ -1,59 +0,0 @@ -import difflib -from pathlib import Path - -import pytest - -import opencc - -_this_dir = Path(__file__).resolve().parent -_opencc_rootdir = _this_dir.parents[1] -_golden_input_dir = _opencc_rootdir / 'test' / 'golden' / 'input' -_golden_output_dir = _opencc_rootdir / 'test' / 'golden' / 'output' - -_GOLDEN_INPUTS = [ - 'us_constitution_zhs.txt', -] - - -def _available_python_golden_cases(): - configs = [ - config[:-5] if config.endswith('.json') else config - for config in opencc.CONFIGS - if 'jieba' not in config - ] - cases = [] - for input_name in _GOLDEN_INPUTS: - input_stem = Path(input_name).stem - for config in configs: - expected_path = _golden_output_dir / f'{input_stem}.{config}.txt' - if expected_path.is_file(): - cases.append((input_name, config, expected_path)) - return cases - - -_GOLDEN_CASES = _available_python_golden_cases() - - -def test_python_golden_cases_are_available(): - assert _GOLDEN_CASES - - -@pytest.mark.parametrize( - 'input_name,config,expected_path', - _GOLDEN_CASES, -) -def test_python_matches_project_golden_output(input_name, config, expected_path): - input_path = _golden_input_dir / input_name - source_text = input_path.read_text(encoding='utf-8') - expected = expected_path.read_text(encoding='utf-8') - - actual = opencc.OpenCC(config).convert(source_text) - - assert actual == expected, ''.join( - difflib.unified_diff( - expected.splitlines(keepends=True), - actual.splitlines(keepends=True), - fromfile=str(expected_path), - tofile=f'opencc.OpenCC({config!r})', - ) - ) From a90522a70c6bfa73f497d2e082c2ec9565dc4e31 Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Sat, 27 Jun 2026 18:33:26 -0700 Subject: [PATCH 07/13] Add opencc-py pure Python package and fix resource zip build Introduce python-pure/ as a parallel installable package (opencc-py) containing the pure Python OpenCC implementation, with resource zip support for generated dictionaries and a clean Bazel build target. Detailed Changes: - **opencc_resources_zip**: - Add dict_zip_name() to preserve subdirectory paths (fixes cngov/ basename collision with main STCharacters.txt). - Add --exclude-dict-subdirs flag to skip dicts in subdirectories, and suppress configs that reference those dicts. - Add exclude_dict_subdirs attribute to the Starlark rule. - Add opencc_py_resources_zip Bazel target using this flag so the opencc-py bundle omits cngov entirely. - **python-pure/opencc/_opencc_pure.py**: - _get_trie: filesystem-first lookup with zip fallback for generated files (STPhrases_GeneratedFromRegionalPhrases, TSCharactersExt). - OpenCC.__init__: filesystem configs take priority over zip; always pass zip_loader for dict fallback; support no-segmentation configs (t2jp, jp2t). - list_configs: scan jieba config dirs even in zip mode; filter out configs referencing subdirectory dicts (_config_uses_subdir_dicts). - _GroupMatcher: preserve group hierarchy; implement union/ short_circuit match_policy. - _ZipLoader: new class for reading configs and dicts from zip. - OpenCC.convert: apply normalization pre-pass before segmentation. - **python-pure/**: package scaffold (setup.py, tests, benchmarks, .gitignore) for the opencc-py distribution. --- data/BUILD.bazel | 11 + data/scripts/opencc_resources_zip.bzl | 3 + data/scripts/opencc_resources_zip.py | 50 +- python-pure/.gitignore | 6 + python-pure/benchmarks/benchmark_opencc.py | 189 +++++ python-pure/opencc/__init__.py | 13 + python-pure/opencc/_opencc_pure.py | 832 +++++++++++++++++++++ python-pure/opencc/py.typed | 0 python-pure/setup.py | 285 +++++++ python-pure/tests/requirements_lock.txt | 6 + python-pure/tests/test_opencc.py | 145 ++++ python-pure/tests/test_opencc_golden.py | 59 ++ 12 files changed, 1597 insertions(+), 2 deletions(-) create mode 100644 python-pure/.gitignore create mode 100644 python-pure/benchmarks/benchmark_opencc.py create mode 100644 python-pure/opencc/__init__.py create mode 100644 python-pure/opencc/_opencc_pure.py create mode 100644 python-pure/opencc/py.typed create mode 100644 python-pure/setup.py create mode 100644 python-pure/tests/requirements_lock.txt create mode 100644 python-pure/tests/test_opencc.py create mode 100644 python-pure/tests/test_opencc_golden.py diff --git a/data/BUILD.bazel b/data/BUILD.bazel index 912505dd1..e654e6354 100644 --- a/data/BUILD.bazel +++ b/data/BUILD.bazel @@ -16,6 +16,17 @@ opencc_resources_zip( out = "opencc-resources.zip", ) +opencc_resources_zip( + name = "opencc_py_resources_zip", + configs = ["//data/config:config"], + dicts = [ + "//data/dictionary:text_dictionaries", + "//data/dictionary:CJK_Compatibility_Ideographs.txt", + ], + exclude_dict_subdirs = True, + out = "opencc-py-resources.zip", +) + opencc_resources_ocd2_zip( name = "opencc_resources_ocd2_zip", src = ":opencc_resources_zip", diff --git a/data/scripts/opencc_resources_zip.bzl b/data/scripts/opencc_resources_zip.bzl index dc67c10d8..4fb218c33 100644 --- a/data/scripts/opencc_resources_zip.bzl +++ b/data/scripts/opencc_resources_zip.bzl @@ -6,6 +6,8 @@ def _opencc_resources_zip_impl(ctx): args.add("--volatile-status", ctx.version_file) args.add_all("--configs", ctx.files.configs) args.add_all("--dicts", ctx.files.dicts) + if ctx.attr.exclude_dict_subdirs: + args.add("--exclude-dict-subdirs") ctx.actions.run( executable = ctx.executable.tool, @@ -25,6 +27,7 @@ opencc_resources_zip = rule( attrs = { "configs": attr.label_list(allow_files = True, mandatory = True), "dicts": attr.label_list(allow_files = True, mandatory = True), + "exclude_dict_subdirs": attr.bool(default = False), "source_url": attr.string(default = "https://github.com/BYVoid/OpenCC"), "tool": attr.label( default = Label("//data/scripts:opencc_resources_zip"), diff --git a/data/scripts/opencc_resources_zip.py b/data/scripts/opencc_resources_zip.py index 040193ba4..a7ad01094 100644 --- a/data/scripts/opencc_resources_zip.py +++ b/data/scripts/opencc_resources_zip.py @@ -24,6 +24,12 @@ def parse_args(): parser.add_argument("--source-url", default=DEFAULT_SOURCE_URL) parser.add_argument("--stable-status") parser.add_argument("--volatile-status") + parser.add_argument( + "--exclude-dict-subdirs", + action="store_true", + default=False, + help="Skip dict files stored under a subdirectory (e.g. cngov/).", + ) return parser.parse_args() @@ -146,6 +152,21 @@ def convert_dict_references(value): return value +def config_references_subdir_dict(value): + """Return True if the config JSON references any dict file under a subdirectory.""" + if isinstance(value, dict): + for key, child in value.items(): + if key == "file" and isinstance(child, str) and "/" in child: + return True + if config_references_subdir_dict(child): + return True + elif isinstance(value, list): + for item in value: + if config_references_subdir_dict(item): + return True + return False + + def read_text_config(path): with open(path, encoding="utf-8") as file: config = json.load(file) @@ -172,6 +193,8 @@ def write_entry(archive, name, content, date_time): def add_entry(archive, entries, name, content, date_time): + if name in entries: + return write_entry(archive, name, content, date_time) entries[name] = { "sha256": hashlib.sha256(content.encode("utf-8")).hexdigest(), @@ -179,6 +202,21 @@ def add_entry(archive, entries, name, content, date_time): } +def dict_zip_name(path): + """Return the name for a dict file inside the zip archive. + + Preserves subdirectory components relative to the nearest 'dictionary' + ancestor so that files in subdirectories (e.g. cngov/STCharacters.txt) + are stored with their subpath and do not collide with top-level files + of the same basename. + """ + parts = path.replace("\\", "/").split("/") + for i in range(len(parts) - 2, -1, -1): + if parts[i] == "dictionary": + return "/".join(parts[i + 1 :]) + return os.path.basename(path) + + def main(): args = parse_args() output_dir = os.path.dirname(args.output) @@ -188,6 +226,11 @@ def main(): with zipfile.ZipFile(args.output, "w") as archive: entries = {} for path in sorted(args.configs, key=os.path.basename): + if args.exclude_dict_subdirs: + with open(path, encoding="utf-8") as f: + raw_cfg = json.load(f) + if config_references_subdir_dict(raw_cfg): + continue add_entry( archive, entries, @@ -195,11 +238,14 @@ def main(): read_text_config(path), date_time, ) - for path in sorted(args.dicts, key=os.path.basename): + for path in sorted(args.dicts, key=dict_zip_name): + name = dict_zip_name(path) + if args.exclude_dict_subdirs and "/" in name: + continue add_entry( archive, entries, - os.path.basename(path), + name, read_clean_dictionary(path), date_time, ) diff --git a/python-pure/.gitignore b/python-pure/.gitignore new file mode 100644 index 000000000..9efeaf1f0 --- /dev/null +++ b/python-pure/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +dist/ +build/ +opencc/opencc-resources*.zip diff --git a/python-pure/benchmarks/benchmark_opencc.py b/python-pure/benchmarks/benchmark_opencc.py new file mode 100644 index 000000000..16067e14b --- /dev/null +++ b/python-pure/benchmarks/benchmark_opencc.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Benchmark the pure Python OpenCC package with project data.""" + +import argparse +import json +import statistics +import sys +import time +from pathlib import Path + +import opencc +from opencc import _opencc_pure + +_this_dir = Path(__file__).resolve().parent +_opencc_rootdir = _this_dir.parents[1] +_testcases_path = _opencc_rootdir / 'test' / 'testcases' / 'testcases.json' +_golden_input_path = _opencc_rootdir / 'test' / 'golden' / 'input' / 'us_constitution_zhs.txt' +_long_input_path = _opencc_rootdir / 'test' / 'benchmark' / 'zuozhuan.txt' + + +def _config_name(config): + return config[:-5] if config.endswith('.json') else config + + +def _default_configs(): + preferred = ['s2t', 't2s', 's2tw', 's2twp', 's2hk', 'tw2s'] + available = { + _config_name(config) + for config in opencc.CONFIGS + if 'jieba' not in config + } + return [config for config in preferred if config in available] + + +def _load_texts(dataset): + if dataset == 'testcases': + with open(_testcases_path, encoding='utf-8') as f: + parsed = json.load(f) + return [ + case['input'] + for case in parsed.get('cases', []) + if case.get('input') + ] + + if dataset == 'golden': + return [_golden_input_path.read_text(encoding='utf-8')] + + if dataset == 'long': + return [_long_input_path.read_text(encoding='utf-8')] + + raise ValueError(f'Unknown dataset: {dataset}') + + +def _time_call(fn): + start = time.perf_counter() + result = fn() + return time.perf_counter() - start, result + + +def _median(values): + return statistics.median(values) if values else 0.0 + + +def _benchmark_init(configs, iterations): + rows = [] + for config in configs: + samples = [] + for _ in range(iterations): + _opencc_pure._trie_cache.clear() + elapsed, _ = _time_call(lambda: opencc.OpenCC(config)) + samples.append(elapsed) + rows.append({ + 'name': f'init/{config}', + 'iterations': iterations, + 'seconds_median': _median(samples), + 'seconds_min': min(samples), + 'seconds_max': max(samples), + 'chars_per_second': None, + }) + return rows + + +def _benchmark_convert(configs, texts, warmups, iterations): + total_chars = sum(len(text) for text in texts) + rows = [] + for config in configs: + converter = opencc.OpenCC(config) + for _ in range(warmups): + for text in texts: + converter.convert(text) + + samples = [] + for _ in range(iterations): + elapsed, _ = _time_call( + lambda: [converter.convert(text) for text in texts] + ) + samples.append(elapsed) + + median = _median(samples) + rows.append({ + 'name': f'convert/{config}', + 'iterations': iterations, + 'seconds_median': median, + 'seconds_min': min(samples), + 'seconds_max': max(samples), + 'chars_per_second': total_chars / median if median > 0 else 0.0, + }) + return rows + + +def _print_table(rows): + print('| benchmark | iterations | median ms | min ms | max ms | chars/s |') + print('|---|---:|---:|---:|---:|---:|') + for row in rows: + chars_per_second = row['chars_per_second'] + if chars_per_second is None: + throughput = '' + else: + throughput = f'{chars_per_second:,.0f}' + print( + '| {name} | {iterations} | {median:.3f} | {min:.3f} | {max:.3f} | {throughput} |'.format( + name=row['name'], + iterations=row['iterations'], + median=row['seconds_median'] * 1000, + min=row['seconds_min'] * 1000, + max=row['seconds_max'] * 1000, + throughput=throughput, + ) + ) + + +def _print_json(rows): + print(json.dumps(rows, ensure_ascii=False, indent=2)) + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument( + '--dataset', + choices=['testcases', 'golden', 'long'], + default='golden', + help='Project dataset to use for conversion benchmarks.', + ) + parser.add_argument( + '--configs', + nargs='+', + default=None, + help='Config names to benchmark. Defaults to common non-jieba configs.', + ) + parser.add_argument( + '--warmups', + type=int, + default=1, + help='Warmup conversion runs before timing.', + ) + parser.add_argument( + '--iterations', + type=int, + default=5, + help='Timed iterations per benchmark.', + ) + parser.add_argument( + '--include-init', + action='store_true', + help='Also benchmark converter construction time.', + ) + parser.add_argument( + '--json', + action='store_true', + help='Emit machine-readable JSON instead of a Markdown table.', + ) + args = parser.parse_args(argv) + + configs = [_config_name(config) for config in (args.configs or _default_configs())] + texts = _load_texts(args.dataset) + + rows = [] + if args.include_init: + rows.extend(_benchmark_init(configs, args.iterations)) + rows.extend(_benchmark_convert(configs, texts, args.warmups, args.iterations)) + + if args.json: + _print_json(rows) + else: + _print_table(rows) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/python-pure/opencc/__init__.py b/python-pure/opencc/__init__.py new file mode 100644 index 000000000..cf7b437bb --- /dev/null +++ b/python-pure/opencc/__init__.py @@ -0,0 +1,13 @@ +from importlib import metadata + +from opencc._opencc_pure import OpenCC +from opencc._opencc_pure import list_configs + +__all__ = ['CONFIGS', 'OpenCC', '__version__'] + +try: + __version__ = metadata.version('opencc-py') +except metadata.PackageNotFoundError: + __version__ = 'dev' + +CONFIGS = list_configs() diff --git a/python-pure/opencc/_opencc_pure.py b/python-pure/opencc/_opencc_pure.py new file mode 100644 index 000000000..32a97f798 --- /dev/null +++ b/python-pure/opencc/_opencc_pure.py @@ -0,0 +1,832 @@ +""" +Pure Python implementation of the OpenCC Chinese conversion algorithm. + +Algorithm overview (mirrors the C++ reference implementation): + + Dictionary (.txt) + Each line: ``keyvalue1 value2 ...`` + The first listed candidate is used as the conversion target. + + Trie + Character-by-character prefix tree. Built per dictionary file. + ``prefix_match(text, pos)`` returns ``(matched_length, value)`` + for the *longest* match starting at ``text[pos]``. + + DictGroup (_GroupMatcher) + An ordered list of tries. At each position the tries are tried in + declaration order; the **first** trie that has any match (not the + globally longest) wins. This gives phrase dictionaries priority + over character dictionaries, mirroring the C++ PrefixMatch logic. + + Conversion + Scans the text left-to-right. At each position the active + group/dict matcher is consulted; the matched substring is replaced + with the dictionary value. On no match, exactly one character is + passed through unchanged. + + MaxMatch Segmentation + Same scan as Conversion, but instead of replacing tokens the + matched keys are emitted as segment boundaries. Unmatched + characters accumulate in a buffer that is flushed as a single + segment when the next match is found or the string ends. + + Conversion Chain + Multiple conversion steps applied sequentially to each segment + produced by the segmenter. Each step uses an independent + group/dict matcher. +""" + +import io +import json +import os +import importlib.util +import zipfile +from pathlib import Path + +__all__ = ['OpenCC'] + +_this_dir = os.path.dirname(os.path.abspath(__file__)) + +# Bundled resource zip (takes priority when present alongside the package). +_pkg_resource_zip = os.path.join(_this_dir, 'opencc-resources.zip') + +# Legacy per-directory layout (kept for Bazel runfile and dev-tree fallback). +_pkg_config_dir = os.path.join(_this_dir, 'config') +_pkg_dict_dir = os.path.join(_this_dir, 'dictionary') +_pkg_jieba_dict_dir = os.path.join(_this_dir, 'jieba_dict') + + +_MAX_PARENT_TRAVERSAL_DEPTH = 8 + + +def _runfile_dir(relative_path: str) -> str: + runfiles_root = os.environ.get('RUNFILES_DIR') + workspace = os.environ.get('TEST_WORKSPACE', '_main') + if not runfiles_root: + return '' + + for candidate in ( + Path(runfiles_root) / workspace / relative_path, + Path(runfiles_root) / relative_path): + if candidate.is_dir(): + return str(candidate) + return '' + + +def _find_repo_root() -> str: + """ + Walk up the directory tree to find the OpenCC repo root. + + Detects the repo by the presence of ``data/config/`` and + ``data/dictionary/`` subdirectories. Returns an empty string when the + repo root cannot be determined (e.g. in a standalone installed package + without bundled data files). + """ + candidate = _this_dir + for _ in range(_MAX_PARENT_TRAVERSAL_DEPTH): + if (os.path.isdir(os.path.join(candidate, 'data', 'config')) and + os.path.isdir(os.path.join(candidate, 'data', 'dictionary'))): + return candidate + parent = os.path.dirname(candidate) + if parent == candidate: + break + candidate = parent + return '' + + +_repo_root = _find_repo_root() +_repo_config_dir = os.path.join(_repo_root, 'data', 'config') if _repo_root else '' +_repo_dict_dir = os.path.join(_repo_root, 'data', 'dictionary') if _repo_root else '' +_repo_jieba_config_dir = os.path.join(_repo_root, 'plugins', 'jieba', 'data', 'config') if _repo_root else '' +_repo_jieba_dict_dir = os.path.join(_repo_root, 'plugins', 'jieba', 'deps', 'cppjieba', 'dict') if _repo_root else '' +_runfiles_config_dir = _runfile_dir('data/config') +_runfiles_dict_dir = _runfile_dir('data/dictionary') +_runfiles_jieba_config_dir = _runfile_dir('plugins/jieba/data/config') +_runfiles_jieba_dict_dir = _runfile_dir('plugins/jieba/deps/cppjieba/dict') + + +# Trie + +class _TrieNode: + """Single node in the character trie.""" + + __slots__ = ('children', 'value') + + def __init__(self): + self.children = {} # char -> _TrieNode + self.value = None # str or None; non-None marks a terminal node + + +class _Trie: + """ + Character-by-character trie for longest-prefix matching. + + Mirrors the inner ``Table`` class in ``PrefixMatch.cpp``. + """ + + __slots__ = ('root',) + + def __init__(self): + self.root = _TrieNode() + + def add(self, key: str, value: str) -> None: + """Insert ``key -> value`` into the trie.""" + node = self.root + for ch in key: + child = node.children.get(ch) + if child is None: + child = _TrieNode() + node.children[ch] = child + node = child + node.value = value + + def prefix_match(self, text: str, pos: int): + """ + Find the longest entry whose key is a prefix of ``text[pos:]``. + + Returns ``(matched_length, value)``; ``matched_length`` is 0 when + no entry matches. + """ + node = self.root + matched_len = 0 + matched_value = None + i = pos + n = len(text) + while i < n: + child = node.children.get(text[i]) + if child is None: + break + node = child + i += 1 + if node.value is not None: + matched_len = i - pos + matched_value = node.value + return matched_len, matched_value + + def match(self, text: str, pos: int): + return self.prefix_match(text, pos) + + +# DictGroup + +class _GroupMatcher: + """ + Ordered collection of sub-matchers with a configurable match policy. + + ``short_circuit`` (default): tries sub-matchers in order; the **first** + sub-matcher that has any match returns its longest result for that entry. + Mirrors ``PrefixMatch`` with a ``short_circuit`` group policy. + + ``union``: tries all sub-matchers and returns the globally longest match + across all of them. Mirrors the ``union`` group policy. + """ + + __slots__ = ('matchers', 'policy') + + def __init__(self, matchers, policy: str = 'short_circuit'): + self.matchers = list(matchers) + self.policy = policy + + def match(self, text: str, pos: int): + """ + Return ``(matched_length, value)`` according to the group policy. + Returns ``(0, None)`` when no sub-matcher has a match. + """ + if self.policy == 'union': + best_len, best_val = 0, None + for m in self.matchers: + length, value = m.match(text, pos) + if length > best_len: + best_len, best_val = length, value + return best_len, best_val + else: # short_circuit + for m in self.matchers: + length, value = m.match(text, pos) + if length > 0: + return length, value + return 0, None + + +# Jieba segmentation + +_jieba_tokenizer_cache: dict = {} + + +class _JiebaSegmenter: + """ + Optional Jieba segmenter backed by the ``jieba`` Python package. + + The dependency is intentionally lazy: importing ``opencc`` and using the + normal mmseg configs never imports or requires ``jieba``. + """ + + __slots__ = ('tokenizer',) + + def __init__(self, resources: dict, config_dir: str = ''): + try: + import jieba + except ImportError as exc: + raise ImportError( + "Config uses Jieba segmentation. Install the optional " + "dependency with `pip install jieba` to use *_jieba configs." + ) from exc + + dict_path = _find_jieba_resource( + resources.get('dict_path', ''), + config_dir, + 'jieba.dict.utf8', + ) + user_dict_path = _find_jieba_resource( + resources.get('user_dict_path', ''), + config_dir, + 'user.dict.utf8', + required=False, + ) + + cache_key = (dict_path, user_dict_path) + tokenizer = _jieba_tokenizer_cache.get(cache_key) + if tokenizer is None: + tokenizer = jieba.Tokenizer(dictionary=dict_path) + tokenizer.initialize() + if user_dict_path: + tokenizer.load_userdict(user_dict_path) + _jieba_tokenizer_cache[cache_key] = tokenizer + self.tokenizer = tokenizer + + def segment(self, text: str) -> list: + return list(self.tokenizer.cut(text, HMM=True)) + + +def _jieba_available() -> bool: + return importlib.util.find_spec('jieba') is not None + + +def _find_jieba_resource(raw_path: str, + config_dir: str, + fallback_name: str, + required: bool = True) -> str: + """ + Resolve Jieba resources for the pure Python optional backend. + + Plugin configs point at ``jieba_merged.ocd2`` for the C++ plugin. Python + ``jieba`` consumes cppjieba's text dictionaries directly, so that request + is mapped to ``jieba.dict.utf8`` plus ``user.dict.utf8``. + """ + candidates = [] + + if raw_path: + if raw_path.endswith('jieba_merged.ocd2'): + candidates.append(raw_path[:-len('jieba_merged.ocd2')] + fallback_name) + candidates.append(fallback_name) + else: + candidates.append(raw_path) + candidates.append(os.path.join(os.path.dirname(raw_path), fallback_name)) + candidates.append(fallback_name) + else: + candidates.append(fallback_name) + + search_dirs = ( + '', + config_dir, + os.path.dirname(config_dir) if config_dir else '', + _pkg_jieba_dict_dir, + _runfiles_jieba_dict_dir, + _repo_jieba_dict_dir, + ) + + seen = set() + for candidate in candidates: + for search_dir in search_dirs: + if not candidate: + continue + path = candidate if os.path.isabs(candidate) or not search_dir else os.path.join(search_dir, candidate) + norm = os.path.abspath(path) + if norm in seen: + continue + seen.add(norm) + if os.path.isfile(path): + return path + + if required: + raise FileNotFoundError(f'Jieba resource not found: {fallback_name!r}') + return '' + + +# Zip-backed resource loader + +class _ZipLoader: + """Read configs and dicts from a flat OpenCC resource zip archive. + + The zip produced by ``//data:opencc_resources_zip`` stores all ``.json`` + and ``.txt`` files at the root level with no subdirectories. + """ + + __slots__ = ('_zf', '_zip_path', '_names') + + def __init__(self, zip_path: str) -> None: + self._zip_path = zip_path + self._zf = zipfile.ZipFile(zip_path, 'r') + self._names: frozenset = frozenset(self._zf.namelist()) + + def has_txt(self, stem: str) -> bool: + return (stem + '.txt') in self._names + + def open_txt(self, stem: str): + """Return a text stream for ``stem.txt``.""" + return io.TextIOWrapper(self._zf.open(stem + '.txt'), encoding='utf-8-sig') + + def has_config(self, name: str) -> bool: + fname = name if name.endswith('.json') else name + '.json' + return fname in self._names + + def read_config(self, name: str) -> dict: + fname = name if name.endswith('.json') else name + '.json' + return json.loads(self._zf.read(fname).decode('utf-8')) + + def list_config_names(self) -> list: + return sorted(n for n in self._names if n.endswith('.json')) + + def cache_key(self, stem: str, reverse: bool) -> tuple: + return (self._zip_path, stem, reverse) + + def close(self) -> None: + self._zf.close() + + def __del__(self) -> None: + try: + self._zf.close() + except Exception: + pass + + +# Dictionary loading + +_trie_cache: dict = {} # cache_key -> _Trie + + +def _find_dict_txt(filename: str, config_dir: str = '', + zip_loader: '_ZipLoader | None' = None): + """ + Resolve a dictionary filename (typically ``Foo.ocd2``) to a + ``(name_or_path, needs_reverse)`` tuple. + + When ``zip_loader`` is given, ``name_or_path`` is the bare stem (e.g. + ``'STPhrases'``) and all I/O goes through the zip. Otherwise it is an + absolute filesystem path. + + Resolution order: + 1. Strip the extension (``Foo.ocd2`` → ``Foo``). + 2. Zip lookup or filesystem search (config dir → pkg dir → runfiles → repo). + 3. If the stem ends with ``Rev`` (e.g. ``HKVariantsRev``), look for + the forward dict (``HKVariants``) and mark it for reversal. + """ + stem = filename + for ext in ('.ocd2', '.ocd', '.txt'): + if stem.endswith(ext): + stem = stem[: -len(ext)] + break + + if zip_loader is not None: + if zip_loader.has_txt(stem): + return stem, False + if stem.endswith('Rev'): + forward_stem = stem[:-3] + if zip_loader.has_txt(forward_stem): + return forward_stem, True + raise FileNotFoundError(f'Dictionary file not found in zip for: {filename!r}') + + if os.path.isabs(filename): + path = stem + '.txt' + if os.path.isfile(path): + return path, False + + search_dirs = ( + config_dir, + _pkg_dict_dir, + _runfiles_dict_dir, + _repo_dict_dir, + ) + + for search_dir in search_dirs: + if not search_dir: + continue + path = os.path.join(search_dir, stem + '.txt') + if os.path.isfile(path): + return path, False + + # Try reversed dict: strip the trailing "Rev" and reverse the forward file. + if stem.endswith('Rev'): + forward_stem = stem[:-3] + for search_dir in search_dirs: + if not search_dir: + continue + path = os.path.join(search_dir, forward_stem + '.txt') + if os.path.isfile(path): + return path, True + + raise FileNotFoundError(f'Dictionary file not found for: {filename!r}') + + +def _parse_dict_lines(name_or_path: str, + zip_loader: '_ZipLoader | None' = None): + """ + Yield ``(key, candidates)`` tuples for every data line in a txt dict file. + + When ``zip_loader`` is given, ``name_or_path`` is a bare stem read from + the zip; otherwise it is a filesystem path. + """ + if zip_loader is not None: + f = zip_loader.open_txt(name_or_path) + else: + f = open(name_or_path, encoding='utf-8-sig') + with f: + for line in f: + line = line.strip() + if not line or line.startswith('#') or '\t' not in line: + continue + key, rest = line.split('\t', 1) + candidates = rest.split() + if candidates: + yield key, candidates + + +def _load_trie(name_or_path: str, reverse: bool, + zip_loader: '_ZipLoader | None' = None) -> _Trie: + """ + Load a ``keyvalue1 value2 ...`` txt file into a :class:`_Trie`. + + When ``reverse`` is ``True`` the mapping is inverted: each candidate + value becomes a key whose conversion target is the original key. When + multiple original keys share the same candidate value the first + encountered (in file order) wins, matching the behaviour of the C++ + dict-reversal script. + """ + if zip_loader is not None: + cache_key = zip_loader.cache_key(name_or_path, reverse) + else: + cache_key = (os.path.abspath(name_or_path), reverse) + + cached = _trie_cache.get(cache_key) + if cached is not None: + return cached + + trie = _Trie() + + if not reverse: + for key, candidates in _parse_dict_lines(name_or_path, zip_loader): + trie.add(key, candidates[0]) + else: + # Reverse: each candidate value -> original key (first writer wins). + reversed_mapping: dict = {} + for key, candidates in _parse_dict_lines(name_or_path, zip_loader): + for candidate in candidates: + if candidate not in reversed_mapping: + reversed_mapping[candidate] = key + for new_key, new_value in reversed_mapping.items(): + trie.add(new_key, new_value) + + _trie_cache[cache_key] = trie + return trie + + +def _get_trie(filename: str, config_dir: str = '', + zip_loader: '_ZipLoader | None' = None) -> _Trie: + """Resolve ``filename`` and return the loaded trie. + + Tries the filesystem first (catches both source and Bazel runfile + locations), then falls back to the resource zip for generated files + that are not present in the source tree. + """ + try: + fs_path, needs_reverse = _find_dict_txt(filename, config_dir, zip_loader=None) + return _load_trie(fs_path, needs_reverse, zip_loader=None) + except FileNotFoundError: + pass + if zip_loader is not None: + stem, needs_reverse = _find_dict_txt(filename, config_dir, zip_loader=zip_loader) + return _load_trie(stem, needs_reverse, zip_loader=zip_loader) + raise FileNotFoundError(f'Dictionary file not found for: {filename!r}') + + +# Config parsing + +def _parse_dict_node(d: dict, config_dir: str = '', + zip_loader: '_ZipLoader | None' = None): + """ + Recursively parse a JSON dict-config node. + + Returns a :class:`_Trie` for leaf nodes or a :class:`_GroupMatcher` for + group nodes, preserving the ``match_policy`` (``short_circuit`` or + ``union``) specified in the config. Mirrors ``ConfigInternal::ParseDict`` + in ``Config.cpp``. + """ + dtype = d['type'] + + if dtype == 'group': + policy = d.get('match_policy', 'short_circuit') + matchers = [_parse_dict_node(sub, config_dir, zip_loader) for sub in d['dicts']] + return _GroupMatcher(matchers, policy=policy) + + if dtype in ('ocd2', 'ocd', 'txt', 'text'): + return _get_trie(d['file'], config_dir, zip_loader) + + raise ValueError(f'Unknown dict type: {dtype!r}') + + +# Core conversion routines + +def _convert_phrase(text: str, matcher: _GroupMatcher) -> str: + """ + Convert *text* using left-to-right longest-prefix matching. + + At each position the group matcher is consulted (first-trie-wins + semantics). On a match the matched substring is replaced by the + dictionary value. On no match exactly one character is passed through + unchanged, mirroring ``Conversion::AppendConverted`` in the C++ code. + """ + parts = [] + i = 0 + n = len(text) + while i < n: + length, value = matcher.match(text, i) + if length > 0: + parts.append(value) + i += length + else: + parts.append(text[i]) + i += 1 + return ''.join(parts) + + +def _segment(text: str, matcher: _GroupMatcher) -> list: + """ + Max-match segmentation of *text*. + + Scans left-to-right using the segmentation dict. On a match the + pending character buffer is flushed as one segment and the matched key + becomes the next segment. Unmatched characters accumulate until the + next match or end of string. + + Mirrors ``MaxMatchSegmentation::Segment`` in the C++ code. + """ + segments = [] + buf = [] + i = 0 + n = len(text) + while i < n: + length, _ = matcher.match(text, i) + if length > 0: + if buf: + segments.append(''.join(buf)) + buf = [] + segments.append(text[i: i + length]) + i += length + else: + buf.append(text[i]) + i += 1 + if buf: + segments.append(''.join(buf)) + return segments + + +# Config location helpers + +def _find_config(config_name: str) -> str: + """ + Locate a config JSON file by name (with or without ``.json`` suffix). + + Searches the bundled package config directory first, then the source-tree + ``data/config`` directory as a fallback. + """ + filename = config_name if config_name.endswith('.json') else config_name + '.json' + for search_dir in ( + _pkg_config_dir, + _runfiles_config_dir, + _repo_config_dir, + _runfiles_jieba_config_dir, + _repo_jieba_config_dir): + if not search_dir: + continue + path = os.path.join(search_dir, filename) + if os.path.isfile(path): + return path + raise FileNotFoundError(f'Config not found: {config_name!r}') + + +def _default_zip_loader() -> '_ZipLoader | None': + """Return a _ZipLoader for the bundled resource zip, or None.""" + env_zip = os.environ.get('OPENCC_RESOURCE_ZIP', '') + if env_zip and os.path.isfile(env_zip): + return _ZipLoader(env_zip) + if os.path.isfile(_pkg_resource_zip): + return _ZipLoader(_pkg_resource_zip) + return None + + +def _config_seg_type(cfg: dict) -> str: + return cfg.get('segmentation', {}).get('type', '') + + +def _config_uses_subdir_dicts(cfg: dict) -> bool: + """Return True if the config references any dict file under a subdirectory.""" + def _check(value) -> bool: + if isinstance(value, dict): + for key, child in value.items(): + if key == 'file' and isinstance(child, str) and '/' in child: + return True + if _check(child): + return True + elif isinstance(value, list): + for item in value: + if _check(item): + return True + return False + return _check(cfg) + + +def _is_supported_cfg(cfg: dict, config_path: str = '') -> bool: + """Return True if the pure Python backend can use this parsed config.""" + seg_type = _config_seg_type(cfg) + if seg_type == 'mmseg': + return True + if seg_type == 'jieba': + if not _jieba_available(): + return False + resources = cfg.get('segmentation', {}).get('resources', {}) + config_dir = os.path.dirname(config_path) if config_path else '' + try: + _find_jieba_resource(resources.get('dict_path', ''), config_dir, 'jieba.dict.utf8') + return True + except FileNotFoundError: + return False + return False + + +def list_configs(resource_zip: 'str | None' = None) -> list: + """ + Return a sorted list of ``.json`` config filenames available to the + pure Python converter. Jieba configs are listed only when the optional + ``jieba`` package and dictionary data are available. + + When ``resource_zip`` is given (or the package ships a bundled + ``opencc-resources.zip``), configs are enumerated from the zip. + """ + zip_loader: '_ZipLoader | None' = None + if resource_zip is not None: + zip_loader = _ZipLoader(resource_zip) + elif resource_zip is None: + zip_loader = _default_zip_loader() + + if zip_loader is not None: + try: + configs = [] + seen: set = set() + for name in zip_loader.list_config_names(): + try: + cfg = zip_loader.read_config(name) + if _is_supported_cfg(cfg): + configs.append(name) + seen.add(name) + except Exception: + pass + # Jieba configs are not bundled in the zip; scan filesystem. + for search_dir in (_runfiles_jieba_config_dir, _repo_jieba_config_dir): + if not search_dir or not os.path.isdir(search_dir): + continue + for fname in sorted(os.listdir(search_dir)): + if not fname.endswith('.json') or fname in seen: + continue + path = os.path.join(search_dir, fname) + try: + with open(path, encoding='utf-8') as f: + cfg = json.load(f) + if _is_supported_cfg(cfg, path) and not _config_uses_subdir_dicts(cfg): + configs.append(fname) + seen.add(fname) + except Exception: + pass + return sorted(configs) + finally: + zip_loader.close() + + configs = [] + seen: set = set() + for search_dir in ( + _pkg_config_dir, + _runfiles_config_dir, + _repo_config_dir, + _runfiles_jieba_config_dir, + _repo_jieba_config_dir): + if not search_dir or not os.path.isdir(search_dir): + continue + for fname in os.listdir(search_dir): + if not fname.endswith('.json') or fname in seen: + continue + path = os.path.join(search_dir, fname) + try: + with open(path, encoding='utf-8') as f: + cfg = json.load(f) + if _is_supported_cfg(cfg, path): + configs.append(fname) + seen.add(fname) + except Exception: + pass + return sorted(configs) + + +# Public API + +class OpenCC: + """ + Pure Python OpenCC converter. + + Usage:: + + cc = OpenCC('s2t') # or 's2t.json' + result = cc.convert('汉字') # -> '漢字' + + The ``config`` argument accepts a conversion name (e.g. ``'s2t'``) or + a bare filename (e.g. ``'s2t.json'``). Custom absolute config paths + are also accepted. + """ + + def __init__(self, config: str = 't2s', + resource_zip: 'str | None' = None) -> None: + config_name = config[:-5] if config.endswith('.json') else config + suffixed_config = config if config.endswith('.json') else f'{config}.json' + + # Determine the active zip loader. + if resource_zip is not None: + zip_loader: '_ZipLoader | None' = _ZipLoader(resource_zip) + else: + zip_loader = _default_zip_loader() + self._zip_loader = zip_loader + + # Load the config JSON. + # Filesystem paths take priority over the zip so that custom configs + # and jieba plugin configs (not bundled in the zip) work correctly. + config_dir = '' + if os.path.isfile(config): + with open(config, encoding='utf-8') as f: + cfg = json.load(f) + config_dir = os.path.dirname(os.path.abspath(config)) + elif os.path.isfile(suffixed_config): + with open(suffixed_config, encoding='utf-8') as f: + cfg = json.load(f) + config_dir = os.path.dirname(os.path.abspath(suffixed_config)) + elif zip_loader is not None and zip_loader.has_config(config_name): + cfg = zip_loader.read_config(config_name) + else: + config_path = _find_config(config_name) + with open(config_path, encoding='utf-8') as f: + cfg = json.load(f) + config_dir = os.path.dirname(config_path) + + self.config = suffixed_config + + segmentation = cfg.get('segmentation', {}) + seg_type = segmentation.get('type', '') + # Dict loading always passes zip_loader as a fallback for generated files + # that are not present in the source tree (e.g. STPhrases_Generated…, TSCharactersExt). + if seg_type == 'mmseg': + self._segmenter = _parse_dict_node(segmentation['dict'], config_dir, zip_loader) + elif seg_type == 'jieba': + self._segmenter = _JiebaSegmenter( + segmentation.get('resources', {}), + config_dir, + ) + elif seg_type == '': + self._segmenter = None # no segmentation; whole text is one segment + else: + raise ValueError( + f"Config {config_name!r} uses segmentation type {seg_type!r} " + f"which is not supported by the pure Python backend." + ) + + self._normalization: list = [ + _parse_dict_node(step['dict'], config_dir, zip_loader) + for step in cfg.get('normalization', []) + ] + + self._chain: list = [ + _parse_dict_node(step['dict'], config_dir, zip_loader) + for step in cfg['conversion_chain'] + ] + + def convert(self, text: str) -> str: + """Convert *text* and return the result.""" + for matcher in self._normalization: + text = _convert_phrase(text, matcher) + if self._segmenter is None: + segments = [text] + elif isinstance(self._segmenter, _JiebaSegmenter): + segments = self._segmenter.segment(text) + else: + segments = _segment(text, self._segmenter) + result = [] + for seg in segments: + converted = seg + for matcher in self._chain: + converted = _convert_phrase(converted, matcher) + result.append(converted) + return ''.join(result) diff --git a/python-pure/opencc/py.typed b/python-pure/opencc/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/python-pure/setup.py b/python-pure/setup.py new file mode 100644 index 000000000..93008f840 --- /dev/null +++ b/python-pure/setup.py @@ -0,0 +1,285 @@ +import json +import os +import re +import shutil +import subprocess + +import setuptools +import setuptools.command.build_py + +_this_dir = os.path.dirname(os.path.abspath(__file__)) +_repo_root = os.path.abspath(os.path.join(_this_dir, '..')) + +_author_file = os.path.join(_repo_root, 'AUTHORS') +_readme_file = os.path.join(_repo_root, 'README.md') +_fallback_version = '1.3.1' + + +def _get_version_from_git(): + try: + raw = subprocess.check_output( + ['git', 'describe', '--tags', '--long', '--always'], + cwd=_repo_root, + stderr=subprocess.DEVNULL, + ).decode('utf-8').strip() + except (OSError, subprocess.CalledProcessError): + return '' + + dirty = '' + for diff_cmd in (['git', 'diff', '--quiet'], ['git', 'diff', '--cached', '--quiet']): + try: + subprocess.check_call( + diff_cmd, + cwd=_repo_root, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except (OSError, subprocess.CalledProcessError): + dirty = '.dirty' + break + + release_match = re.match(r'^(?:v|ver\.)(\d+\.\d+\.\d+)-0-g[0-9a-f]+$', raw) + if release_match: + return '{}{}'.format(release_match.group(1), dirty) + + dev_match = re.match(r'^(?:v|ver\.)(\d+\.\d+\.\d+)-(\d+)-g([0-9a-f]+)$', raw) + if dev_match: + return '{}.dev{}+g{}{}'.format( + dev_match.group(1), + dev_match.group(2), + dev_match.group(3), + dirty, + ) + + return '{}+g{}{}'.format(_fallback_version, raw, dirty) + + +def get_version_info(): + env_version = os.environ.get('VERSION', '') + if env_version: + return env_version.lstrip('v') + git_version = _get_version_from_git() + if git_version: + return git_version + return _fallback_version + + +def get_author_info(): + if not os.path.isfile(_author_file): + return 'BYVoid', 'byvoid@byvoid.com' + + authors = [] + emails = [] + author_pattern = re.compile(r'(.+) <(.+)>') + with open(_author_file, 'rb') as f: + for line in f: + match = author_pattern.search(line.decode('utf-8')) + if not match: + continue + authors.append(match.group(1)) + emails.append(match.group(2)) + if not authors: + return 'BYVoid', 'byvoid@byvoid.com' + return ', '.join(authors), ', '.join(emails) + + +def get_long_description(): + if not os.path.isfile(_readme_file): + return '' + with open(_readme_file, encoding='utf-8') as f: + return f.read() + + +def _load_trie_from_txt(path): + """Build a minimal {key: first_value} prefix trie from a .txt dict file.""" + trie = {} + with open(path, encoding='utf-8-sig') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#') or '\t' not in line: + continue + key, rest = line.split('\t', 1) + value = rest.split()[0] + node = trie + for i, ch in enumerate(key): + if ch not in node: + node[ch] = {} + node = node[ch] + node.setdefault('__v__', value) + return trie + + +def _trie_match(trie, text, pos): + """Longest prefix match; returns (length, value) or (0, None).""" + node = trie + best_len, best_val = 0, None + i = pos + while i < len(text): + node = node.get(text[i]) + if node is None: + break + i += 1 + if '__v__' in node: + best_len, best_val = i - pos, node['__v__'] + return best_len, best_val + + +def _t2s_convert(text, tries): + """Convert traditional text to simplified using an ordered list of tries + (short-circuit: first trie that matches at a position wins).""" + result = [] + pos = 0 + while pos < len(text): + matched = False + for trie in tries: + length, value = _trie_match(trie, text, pos) + if length > 0: + result.append(value) + pos += length + matched = True + break + if not matched: + result.append(text[pos]) + pos += 1 + return ''.join(result) + + +def _generate_st_phrases_from_regional(src_dict, dst_dict): + """Generate STPhrases_GeneratedFromRegionalPhrases.txt in dst_dict. + + Mirrors the logic of data/scripts/generate_st_phrases_from_regional_phrases.py: + convert TWPhrases/HKPhrases keys via t2s and emit keys with len >= 3. + """ + ts_phrases_path = os.path.join(src_dict, 'TSPhrases.txt') + ts_chars_path = os.path.join(src_dict, 'TSCharacters.txt') + if not os.path.isfile(ts_phrases_path) or not os.path.isfile(ts_chars_path): + return + + tries = [_load_trie_from_txt(ts_phrases_path), + _load_trie_from_txt(ts_chars_path)] + + collisions = {} # simplified_key -> first_traditional_key + for fname in ('TWPhrases.txt', 'HKPhrases.txt'): + path = os.path.join(src_dict, fname) + if not os.path.isfile(path): + continue + with open(path, encoding='utf-8-sig') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#') or '\t' not in line: + continue + original_key = line.split('\t', 1)[0] + simplified_key = _t2s_convert(original_key, tries) + if len(simplified_key) < 3: + continue + collisions.setdefault(simplified_key, original_key) + + out_path = os.path.join(dst_dict, 'STPhrases_GeneratedFromRegionalPhrases.txt') + with open(out_path, 'w', encoding='utf-8', newline='\n') as f: + f.write('# Open Chinese Convert (OpenCC) Dictionary\n') + f.write('# File: STPhrases_GeneratedFromRegionalPhrases.txt\n') + f.write('# Format: key\tvalue(s) (values separated by spaces)\n') + f.write('# License: Apache-2.0 (see LICENSE)\n') + f.write('# Source: generated from TWPhrases.txt, HKPhrases.txt keys via t2s\n') + f.write('# Used in configs: s2t.json, s2hk.json, s2tw.json\n') + f.write('#\n') + f.write('\n') + for key in sorted(collisions): + f.write('{}\t{}\n'.format(key, collisions[key])) + + +class BuildPyCommand(setuptools.command.build_py.build_py, object): + """Bundle OpenCC's JSON configs and text dictionaries into the package.""" + + def run(self): + super(BuildPyCommand, self).run() + self._copy_opencc_data() + + def _copy_opencc_data(self): + src_config = os.path.join(_repo_root, 'data', 'config') + src_dict = os.path.join(_repo_root, 'data', 'dictionary') + src_jieba_config = os.path.join(_repo_root, 'plugins', 'jieba', 'data', 'config') + src_jieba_dict = os.path.join(_repo_root, 'plugins', 'jieba', 'deps', 'cppjieba', 'dict') + dst_base = os.path.join(self.build_lib, 'opencc') + dst_config = os.path.join(dst_base, 'config') + dst_dict = os.path.join(dst_base, 'dictionary') + dst_jieba_dict = os.path.join(dst_base, 'jieba_dict') + + os.makedirs(dst_config, exist_ok=True) + for fname in os.listdir(src_config): + if not fname.endswith('.json'): + continue + src_config_path = os.path.join(src_config, fname) + with open(src_config_path, encoding='utf-8') as f: + config = json.load(f) + if config.get('segmentation', {}).get('type') != 'mmseg': + continue + shutil.copy2(src_config_path, os.path.join(dst_config, fname)) + if os.path.isdir(src_jieba_config): + for fname in os.listdir(src_jieba_config): + if fname.endswith('.json'): + shutil.copy2( + os.path.join(src_jieba_config, fname), + os.path.join(dst_config, fname), + ) + + os.makedirs(dst_dict, exist_ok=True) + for fname in os.listdir(src_dict): + if fname.endswith('.txt'): + shutil.copy2( + os.path.join(src_dict, fname), + os.path.join(dst_dict, fname), + ) + + _generate_st_phrases_from_regional(src_dict, dst_dict) + + os.makedirs(dst_jieba_dict, exist_ok=True) + for fname in ('jieba.dict.utf8', 'user.dict.utf8'): + src_path = os.path.join(src_jieba_dict, fname) + if os.path.isfile(src_path): + shutil.copy2(src_path, os.path.join(dst_jieba_dict, fname)) + + +version_info = get_version_info() +author_info = get_author_info() + +setuptools.setup( + name='opencc-py', + version=version_info, + author=author_info[0], + author_email=author_info[1], + description='Conversion between Traditional and Simplified Chinese (pure Python)', + long_description=get_long_description(), + long_description_content_type='text/markdown', + url='https://github.com/BYVoid/OpenCC', + + packages=['opencc'], + package_dir={'opencc': 'opencc'}, + package_data={ + 'opencc': ['py.typed', 'config/*.json', 'dictionary/*.txt', 'jieba_dict/*.utf8'], + }, + cmdclass={ + 'build_py': BuildPyCommand, + }, + + python_requires='>=3.8', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'Natural Language :: Chinese (Simplified)', + 'Natural Language :: Chinese (Traditional)', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: Apache Software License', + 'Topic :: Scientific/Engineering', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Software Development :: Localization', + 'Topic :: Text Processing :: Linguistic', + 'Typing :: Typed', + ], + license='Apache License 2.0', + keywords=['opencc', 'convert', 'chinese'], +) diff --git a/python-pure/tests/requirements_lock.txt b/python-pure/tests/requirements_lock.txt new file mode 100644 index 000000000..47e5801dd --- /dev/null +++ b/python-pure/tests/requirements_lock.txt @@ -0,0 +1,6 @@ +exceptiongroup==1.2.2 +iniconfig==2.0.0 +packaging==24.1 +pluggy==1.5.0 +pygments==2.20.0 +pytest==9.0.3 diff --git a/python-pure/tests/test_opencc.py b/python-pure/tests/test_opencc.py new file mode 100644 index 000000000..6b9c393de --- /dev/null +++ b/python-pure/tests/test_opencc.py @@ -0,0 +1,145 @@ +import json +import importlib.util +import os +import pytest +import sys + +_this_dir = os.path.dirname(os.path.abspath(__file__)) +_opencc_rootdir = os.path.abspath(os.path.join(_this_dir, '..', '..')) +_testcases_path = os.path.join(_opencc_rootdir, 'test', 'testcases', 'testcases.json') +_config_test_path = os.path.join(_opencc_rootdir, 'test', 'config_test', 'config_test.json') +_jieba_testcases_path = os.path.join( + _opencc_rootdir, + 'plugins', + 'jieba', + 'tests', + 'data', + 'jieba_comparison_testcases.json', +) + + +def _has_jieba(): + return importlib.util.find_spec('jieba') is not None + + +def test_import(): + import opencc # noqa + + +def test_init_delete_converter(): + import opencc + + for config in opencc.CONFIGS: + converter = opencc.OpenCC(config) + del converter + + +def test_jieba_configs_are_exposed_only_when_optional_dependency_is_available(): + import opencc + + if _has_jieba(): + assert any('jieba' in config for config in opencc.CONFIGS) + else: + assert all('jieba' not in config for config in opencc.CONFIGS) + + +def test_jieba_config_requires_optional_dependency_when_missing(): + if _has_jieba(): + pytest.skip('jieba is installed') + + import opencc + + with pytest.raises(ImportError, match='pip install jieba'): + opencc.OpenCC('s2t_jieba') + + +@pytest.mark.skipif(not _has_jieba(), reason='jieba optional dependency is not installed') +def test_jieba_configs_match_plugin_comparison_cases(): + import opencc + + with open(_jieba_testcases_path, 'r', encoding='utf-8') as f: + parsed = json.load(f) + + for case in parsed['cases']: + for cfg, ans in case.get('expected', {}).items(): + if not cfg.endswith('_jieba'): + continue + converter = opencc.OpenCC(f'{cfg}.json') + assert converter.convert(case['input']) == ans, \ + 'Failed to convert {} for {} -> {}'.format(cfg, case['input'], ans) + + +def test_custom_config_resolves_local_dictionaries(): + import opencc + + converter = opencc.OpenCC(_config_test_path) + + assert converter.convert('燕燕于飞，之子于归。') == '燕燕于飛，之子于歸。' + assert converter.convert('远') == '遠' + + +def test_custom_config_path_accepts_suffixless_json_path(): + import opencc + + converter = opencc.OpenCC(_config_test_path[:-5]) + + assert converter.convert('燕燕于飞，之子于归。') == '燕燕于飛，之子于歸。' + + +def test_custom_config_resolves_absolute_ocd2_path_to_text_dictionary(tmp_path): + import opencc + + dict_txt = tmp_path / 'CustomPhrases.txt' + dict_txt.write_text('测试\t測試\n', encoding='utf-8') + config_path = tmp_path / 'custom.json' + config_path.write_text(json.dumps({ + 'name': 'Custom', + 'segmentation': { + 'type': 'mmseg', + 'dict': { + 'type': 'ocd2', + 'file': str(tmp_path / 'CustomPhrases.ocd2'), + }, + }, + 'conversion_chain': [{ + 'dict': { + 'type': 'ocd2', + 'file': str(tmp_path / 'CustomPhrases.ocd2'), + }, + }], + }), encoding='utf-8') + + converter = opencc.OpenCC(str(config_path)) + + assert converter.convert('测试') == '測試' + + +def test_dictionary_parser_strips_utf8_bom(tmp_path): + from opencc import _opencc_pure + + dict_txt = tmp_path / 'BomPhrases.txt' + dict_txt.write_text('\ufeff测试\t測試\n', encoding='utf-8') + trie = _opencc_pure._load_trie(str(dict_txt), reverse=False) + + assert trie.prefix_match('测试', 0) == (2, '測試') + + +def test_conversion(): + import opencc + + with open(_testcases_path, 'r', encoding='utf-8') as f: + parsed = json.load(f) + + for case in parsed.get('cases', []): + input_text = case.get('input') + expected = case.get('expected', {}) + if not input_text or not isinstance(expected, dict): + continue + for cfg, ans in expected.items(): + converter = opencc.OpenCC(f'{cfg}.json') + assert converter.convert(input_text) == ans, \ + 'Failed to convert {} for {} -> {}'.format(cfg, input_text, ans) + + +if __name__ == "__main__": + sys.exit(pytest.main(sys.argv[1:])) diff --git a/python-pure/tests/test_opencc_golden.py b/python-pure/tests/test_opencc_golden.py new file mode 100644 index 000000000..71c50b194 --- /dev/null +++ b/python-pure/tests/test_opencc_golden.py @@ -0,0 +1,59 @@ +import difflib +from pathlib import Path + +import pytest + +import opencc + +_this_dir = Path(__file__).resolve().parent +_opencc_rootdir = _this_dir.parents[1] +_golden_input_dir = _opencc_rootdir / 'test' / 'golden' / 'input' +_golden_output_dir = _opencc_rootdir / 'test' / 'golden' / 'output' + +_GOLDEN_INPUTS = [ + 'us_constitution_zhs.txt', +] + + +def _available_python_golden_cases(): + configs = [ + config[:-5] if config.endswith('.json') else config + for config in opencc.CONFIGS + if 'jieba' not in config + ] + cases = [] + for input_name in _GOLDEN_INPUTS: + input_stem = Path(input_name).stem + for config in configs: + expected_path = _golden_output_dir / f'{input_stem}.{config}.txt' + if expected_path.is_file(): + cases.append((input_name, config, expected_path)) + return cases + + +_GOLDEN_CASES = _available_python_golden_cases() + + +def test_python_golden_cases_are_available(): + assert _GOLDEN_CASES + + +@pytest.mark.parametrize( + 'input_name,config,expected_path', + _GOLDEN_CASES, +) +def test_python_matches_project_golden_output(input_name, config, expected_path): + input_path = _golden_input_dir / input_name + source_text = input_path.read_text(encoding='utf-8') + expected = expected_path.read_text(encoding='utf-8') + + actual = opencc.OpenCC(config).convert(source_text) + + assert actual == expected, ''.join( + difflib.unified_diff( + expected.splitlines(keepends=True), + actual.splitlines(keepends=True), + fromfile=str(expected_path), + tofile=f'opencc.OpenCC({config!r})', + ) + ) From 0a4e15f5f7b6b5228ee4dfa3441cc90ac2c4cdb9 Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Sat, 27 Jun 2026 19:30:03 -0700 Subject: [PATCH 08/13] Honour may_output_tofu flag in pure Python converter `_parse_dict_node` now accepts `include_tofu_risk_dicts` and returns `None` for any dict node whose `may_output_tofu` is true when the flag is false, mirroring `ConfigInternal::ParseDict` in Config.cpp. `OpenCC.__init__` gains `include_tofu_risk_dictionaries` (default `True`, matching the C++ Python binding); jieba comparison tests pass `False` explicitly since those golden values are CLI outputs, where tofu-risk dictionaries are excluded by default. --- python-pure/opencc/_opencc_pure.py | 35 ++++++++++++++++++++---------- python-pure/tests/test_opencc.py | 4 +++- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/python-pure/opencc/_opencc_pure.py b/python-pure/opencc/_opencc_pure.py index 32a97f798..d9056264f 100644 --- a/python-pure/opencc/_opencc_pure.py +++ b/python-pure/opencc/_opencc_pure.py @@ -511,22 +511,31 @@ def _get_trie(filename: str, config_dir: str = '', # Config parsing def _parse_dict_node(d: dict, config_dir: str = '', - zip_loader: '_ZipLoader | None' = None): + zip_loader: '_ZipLoader | None' = None, + include_tofu_risk_dicts: bool = True): """ Recursively parse a JSON dict-config node. - Returns a :class:`_Trie` for leaf nodes or a :class:`_GroupMatcher` for - group nodes, preserving the ``match_policy`` (``short_circuit`` or - ``union``) specified in the config. Mirrors ``ConfigInternal::ParseDict`` - in ``Config.cpp``. + Returns a :class:`_Trie` for leaf nodes, a :class:`_GroupMatcher` for + group nodes, or ``None`` when a dict is suppressed because + ``may_output_tofu`` is true and ``include_tofu_risk_dicts`` is false. + Mirrors ``ConfigInternal::ParseDict`` in ``Config.cpp``. """ dtype = d['type'] if dtype == 'group': policy = d.get('match_policy', 'short_circuit') - matchers = [_parse_dict_node(sub, config_dir, zip_loader) for sub in d['dicts']] + matchers = [ + m for sub in d['dicts'] + if (m := _parse_dict_node(sub, config_dir, zip_loader, include_tofu_risk_dicts)) is not None + ] + if not matchers: + return None return _GroupMatcher(matchers, policy=policy) + if d.get('may_output_tofu', False) and not include_tofu_risk_dicts: + return None + if dtype in ('ocd2', 'ocd', 'txt', 'text'): return _get_trie(d['file'], config_dir, zip_loader) @@ -751,7 +760,8 @@ class OpenCC: """ def __init__(self, config: str = 't2s', - resource_zip: 'str | None' = None) -> None: + resource_zip: 'str | None' = None, + include_tofu_risk_dictionaries: bool = True) -> None: config_name = config[:-5] if config.endswith('.json') else config suffixed_config = config if config.endswith('.json') else f'{config}.json' @@ -786,10 +796,11 @@ def __init__(self, config: str = 't2s', segmentation = cfg.get('segmentation', {}) seg_type = segmentation.get('type', '') + include_tofu = include_tofu_risk_dictionaries # Dict loading always passes zip_loader as a fallback for generated files # that are not present in the source tree (e.g. STPhrases_Generated…, TSCharactersExt). if seg_type == 'mmseg': - self._segmenter = _parse_dict_node(segmentation['dict'], config_dir, zip_loader) + self._segmenter = _parse_dict_node(segmentation['dict'], config_dir, zip_loader, include_tofu) elif seg_type == 'jieba': self._segmenter = _JiebaSegmenter( segmentation.get('resources', {}), @@ -804,13 +815,13 @@ def __init__(self, config: str = 't2s', ) self._normalization: list = [ - _parse_dict_node(step['dict'], config_dir, zip_loader) - for step in cfg.get('normalization', []) + m for step in cfg.get('normalization', []) + if (m := _parse_dict_node(step['dict'], config_dir, zip_loader, include_tofu)) is not None ] self._chain: list = [ - _parse_dict_node(step['dict'], config_dir, zip_loader) - for step in cfg['conversion_chain'] + m for step in cfg['conversion_chain'] + if (m := _parse_dict_node(step['dict'], config_dir, zip_loader, include_tofu)) is not None ] def convert(self, text: str) -> str: diff --git a/python-pure/tests/test_opencc.py b/python-pure/tests/test_opencc.py index 6b9c393de..68c2ee0a3 100644 --- a/python-pure/tests/test_opencc.py +++ b/python-pure/tests/test_opencc.py @@ -64,7 +64,9 @@ def test_jieba_configs_match_plugin_comparison_cases(): for cfg, ans in case.get('expected', {}).items(): if not cfg.endswith('_jieba'): continue - converter = opencc.OpenCC(f'{cfg}.json') + # Jieba comparison testcases record CLI golden outputs, which skip + # tofu-risk dictionaries by default (same as --include-tofu-risk-dictionaries=false). + converter = opencc.OpenCC(f'{cfg}.json', include_tofu_risk_dictionaries=False) assert converter.convert(case['input']) == ans, \ 'Failed to convert {} for {} -> {}'.format(cfg, case['input'], ans) From a3cd91140a12637473877d8a3d57af22c799ec98 Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Sun, 28 Jun 2026 06:18:58 -0700 Subject: [PATCH 09/13] Load pure Python resources from opencc-data Replace source-tree and bundled-resource data discovery with the opencc-data runtime dependency so the pure Python package no longer depends on generated OpenCC artifacts. Keep the opencc-py package version and its pinned opencc-data dependency version identical for data compatibility. Detailed Changes: - **Resource Loading**: - Load built-in configs and dictionaries through opencc_data resources. - Keep custom config support for local text dictionaries next to user configs. - **Packaging and License**: - Set the opencc-py version to match the opencc-data version. - Pin opencc-data to the identical runtime dependency version. - Set package author metadata to OpenCC contributors. - Include an Apache-2.0 LICENSE file in python-pure package builds. - **Documentation and Tests**: - Add English and Traditional Chinese README files with cross-links. - Link to the official OpenCC project and the opencc-data PyPI package. - Document that this package is an OpenCC derivative work. - Update pure Python tests to use opencc-data test fixtures. --- python-pure/LICENSE | 55 ++++ python-pure/README.md | 113 ++++++++ python-pure/README.zh-TW.md | 103 +++++++ python-pure/opencc/_opencc_pure.py | 369 ++++++------------------ python-pure/setup.py | 249 +--------------- python-pure/tests/requirements_lock.txt | 1 + python-pure/tests/test_opencc.py | 55 +--- 7 files changed, 376 insertions(+), 569 deletions(-) create mode 100644 python-pure/LICENSE create mode 100644 python-pure/README.md create mode 100644 python-pure/README.zh-TW.md diff --git a/python-pure/LICENSE b/python-pure/LICENSE new file mode 100644 index 000000000..1914f856a --- /dev/null +++ b/python-pure/LICENSE @@ -0,0 +1,55 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + 1. You must give any other recipients of the Work or Derivative Works a copy of this License; and + + 2. You must cause any modified files to carry prominent notices stating that You changed the files; and + + 3. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + 4. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/python-pure/README.md b/python-pure/README.md new file mode 100644 index 000000000..435b5fdec --- /dev/null +++ b/python-pure/README.md @@ -0,0 +1,113 @@ +# OpenCC Pure Python + +[繁體中文版](README.zh-TW.md) + +This directory contains a pure Python implementation of the +[OpenCC](https://github.com/BYVoid/OpenCC) Chinese conversion algorithm. It +provides the same import surface as the Python package: + +```python +import opencc + +converter = opencc.OpenCC("s2t") +print(converter.convert("汉字")) # 漢字 +``` + +## Data Dependency + +The package does not bundle OpenCC configs or dictionaries directly. Built-in +conversion data is loaded from the +[`opencc-data` PyPI package](https://pypi.org/project/opencc-data/) at runtime. + +This keeps the pure Python package small and avoids depending on generated files +under the OpenCC source tree. The converter reads: + +- config JSON files from `opencc_data.config_path()` +- dictionary text files from `opencc_data.data_path()` +- test cases from `opencc_data.test_data_path()` + +Custom config files are still supported. When a custom config references a local +dictionary path such as `CustomPhrases.ocd2`, the pure Python implementation +looks for the corresponding `CustomPhrases.txt` next to the config file. + +## Installation + +Install from this directory with pip: + +```bash +python -m pip install . +``` + +The package version matches its `opencc-data` version and declares the matching +data package as an exact install dependency, so pip installs the compatible data +package automatically. + +For editable development: + +```bash +python -m pip install -e . +``` + +## Supported Configs + +`opencc.CONFIGS` is populated from the configs exposed by `opencc-data`. + +```python +import opencc + +print(opencc.CONFIGS) +``` + +The standard mmseg configs and configs that do not require segmentation are +supported. Jieba plugin configs are not included in `opencc-data`, so they are +not exposed as built-in configs by this package. + +## Testing + +Install test dependencies, then run pytest from the repository root: + +```bash +python -m pip install -r python-pure/tests/requirements_lock.txt +PYTHONPATH=python-pure python -m pytest python-pure/tests +``` + +The tests verify: + +- importing and initializing every built-in config +- conversion against `opencc-data` test cases +- custom config and local dictionary resolution +- golden output compatibility for supported configs + +## Differences from the Official Implementation + +This package intentionally implements only the pieces needed for pure Python +text conversion. Compared with the official C++ library and command-line tools, +it omits several lower-level details: + +- binary dictionary loading for `.ocd2`/`.ocd`; built-in dictionaries are read + from `.txt` data supplied by `opencc-data` +- dictionary compilation and extraction tools such as `opencc_dict` and + `opencc_phrase_extract` +- the C API, shared-library loading behavior, and ABI/plugin compatibility + guarantees +- native CLI behavior, including streaming I/O, command-line option parity, and + platform-specific path handling +- package, runfiles, and source-tree data discovery fallbacks; built-in data + comes from `opencc-data` +- automatic loading of optional plugin configs or plugin resources, including + the Jieba plugin package layout +- performance optimizations from marisa-trie, Darts, and the C++ segmentation + implementation + +The conversion semantics still mirror OpenCC's config-driven pipeline: mmseg +segmentation, ordered dictionary groups, longest-prefix matching within a +dictionary, conversion chains, normalization, and optional suppression of +tofu-risk dictionaries. + +## License and Compliance + +This package is distributed under the Apache License 2.0. + +This project is a derivative work of +[OpenCC](https://github.com/BYVoid/OpenCC). Runtime conversion data is provided +by the [`opencc-data` PyPI package](https://pypi.org/project/opencc-data/). diff --git a/python-pure/README.zh-TW.md b/python-pure/README.zh-TW.md new file mode 100644 index 000000000..a4a27a669 --- /dev/null +++ b/python-pure/README.zh-TW.md @@ -0,0 +1,103 @@ +# OpenCC Pure Python + +[English version](README.md) + +此目錄包含 [OpenCC](https://github.com/BYVoid/OpenCC) 中文轉換演算法的純 +Python 實作，提供與 Python package 相同的匯入介面： + +```python +import opencc + +converter = opencc.OpenCC("s2t") +print(converter.convert("汉字")) # 漢字 +``` + +## 資料依賴 + +此 package 不再直接內嵌 OpenCC config 或 dictionary。內建轉換資料會在執行時 +從 PyPI package [`opencc-data`](https://pypi.org/project/opencc-data/) 載入。 + +這能讓 pure Python package 保持精簡，並避免依賴 OpenCC source tree 底下的 +生成檔案。converter 會讀取： + +- `opencc_data.config_path()` 提供的 config JSON 檔案 +- `opencc_data.data_path()` 提供的 dictionary text 檔案 +- `opencc_data.test_data_path()` 提供的測試案例 + +自訂 config 仍然支援。當自訂 config 參照本地 dictionary 路徑，例如 +`CustomPhrases.ocd2`，純 Python 實作會在 config 檔案旁尋找對應的 +`CustomPhrases.txt`。 + +## 安裝 + +在此目錄中使用 pip 安裝： + +```bash +python -m pip install . +``` + +此 package 的版本會與 `opencc-data` 版本一致，並將相同版本的資料 package +宣告為精確安裝依賴，因此 pip 會自動安裝相容的資料 package。 + +開發時可使用 editable 安裝： + +```bash +python -m pip install -e . +``` + +## 支援的 Configs + +`opencc.CONFIGS` 由 `opencc-data` 提供的 configs 產生。 + +```python +import opencc + +print(opencc.CONFIGS) +``` + +標準 mmseg configs 與不需要 segmentation 的 configs 皆受支援。Jieba plugin +configs 不包含在 `opencc-data` 中，因此此 package 不會把它們列為內建 configs。 + +## 測試 + +先安裝測試依賴，再從 repository root 執行 pytest： + +```bash +python -m pip install -r python-pure/tests/requirements_lock.txt +PYTHONPATH=python-pure python -m pytest python-pure/tests +``` + +測試會驗證： + +- 每個內建 config 都能 import 與初始化 +- 轉換結果符合 `opencc-data` 測試案例 +- 自訂 config 與本地 dictionary 解析 +- 支援 configs 的 golden output 相容性 + +## 與官方實作的差異 + +此 package 刻意只實作純 Python 文字轉換所需的部分。相較於官方 C++ library +與 command-line tools，它省略了幾個較底層的實作細節： + +- `.ocd2` / `.ocd` 二進位 dictionary 載入；內建 dictionary 會讀取 + `opencc-data` 提供的 `.txt` 資料 +- `opencc_dict`、`opencc_phrase_extract` 等 dictionary 編譯與抽取工具 +- C API、shared-library 載入行為，以及 ABI/plugin 相容性保證 +- native CLI 行為，包括 streaming I/O、命令列選項完整對齊，以及平台相關路徑處理 +- package、runfiles、source-tree 資料搜尋 fallback；內建資料一律來自 + `opencc-data` +- optional plugin configs 或 plugin resources 的自動載入，包括 Jieba plugin 的 + package layout +- marisa-trie、Darts 與 C++ segmentation 實作帶來的效能最佳化 + +轉換語意仍會對齊 OpenCC 的 config-driven pipeline：mmseg segmentation、 +ordered dictionary groups、dictionary 內 longest-prefix matching、conversion +chains、normalization，以及 tofu-risk dictionaries 的可選停用。 + +## License 與合規 + +此 package 以 Apache License 2.0 發佈。 + +此專案屬於 [OpenCC](https://github.com/BYVoid/OpenCC) 的衍生作品。執行時轉換 +資料由 PyPI package [`opencc-data`](https://pypi.org/project/opencc-data/) +提供。 diff --git a/python-pure/opencc/_opencc_pure.py b/python-pure/opencc/_opencc_pure.py index d9056264f..fd833704f 100644 --- a/python-pure/opencc/_opencc_pure.py +++ b/python-pure/opencc/_opencc_pure.py @@ -36,73 +36,27 @@ group/dict matcher. """ -import io import json import os import importlib.util -import zipfile -from pathlib import Path __all__ = ['OpenCC'] -_this_dir = os.path.dirname(os.path.abspath(__file__)) +_opencc_data = None -# Bundled resource zip (takes priority when present alongside the package). -_pkg_resource_zip = os.path.join(_this_dir, 'opencc-resources.zip') -# Legacy per-directory layout (kept for Bazel runfile and dev-tree fallback). -_pkg_config_dir = os.path.join(_this_dir, 'config') -_pkg_dict_dir = os.path.join(_this_dir, 'dictionary') -_pkg_jieba_dict_dir = os.path.join(_this_dir, 'jieba_dict') - - -_MAX_PARENT_TRAVERSAL_DEPTH = 8 - - -def _runfile_dir(relative_path: str) -> str: - runfiles_root = os.environ.get('RUNFILES_DIR') - workspace = os.environ.get('TEST_WORKSPACE', '_main') - if not runfiles_root: - return '' - - for candidate in ( - Path(runfiles_root) / workspace / relative_path, - Path(runfiles_root) / relative_path): - if candidate.is_dir(): - return str(candidate) - return '' - - -def _find_repo_root() -> str: - """ - Walk up the directory tree to find the OpenCC repo root. - - Detects the repo by the presence of ``data/config/`` and - ``data/dictionary/`` subdirectories. Returns an empty string when the - repo root cannot be determined (e.g. in a standalone installed package - without bundled data files). - """ - candidate = _this_dir - for _ in range(_MAX_PARENT_TRAVERSAL_DEPTH): - if (os.path.isdir(os.path.join(candidate, 'data', 'config')) and - os.path.isdir(os.path.join(candidate, 'data', 'dictionary'))): - return candidate - parent = os.path.dirname(candidate) - if parent == candidate: - break - candidate = parent - return '' - - -_repo_root = _find_repo_root() -_repo_config_dir = os.path.join(_repo_root, 'data', 'config') if _repo_root else '' -_repo_dict_dir = os.path.join(_repo_root, 'data', 'dictionary') if _repo_root else '' -_repo_jieba_config_dir = os.path.join(_repo_root, 'plugins', 'jieba', 'data', 'config') if _repo_root else '' -_repo_jieba_dict_dir = os.path.join(_repo_root, 'plugins', 'jieba', 'deps', 'cppjieba', 'dict') if _repo_root else '' -_runfiles_config_dir = _runfile_dir('data/config') -_runfiles_dict_dir = _runfile_dir('data/dictionary') -_runfiles_jieba_config_dir = _runfile_dir('plugins/jieba/data/config') -_runfiles_jieba_dict_dir = _runfile_dir('plugins/jieba/deps/cppjieba/dict') +def _get_opencc_data(): + global _opencc_data + if _opencc_data is None: + try: + import opencc_data + except ImportError as exc: + raise ImportError( + 'opencc-py pure Python resources are provided by the ' + '`opencc-data` package. Install it with `pip install opencc-data`.' + ) from exc + _opencc_data = opencc_data + return _opencc_data # Trie @@ -289,9 +243,6 @@ def _find_jieba_resource(raw_path: str, '', config_dir, os.path.dirname(config_dir) if config_dir else '', - _pkg_jieba_dict_dir, - _runfiles_jieba_dict_dir, - _repo_jieba_dict_dir, ) seen = set() @@ -312,51 +263,25 @@ def _find_jieba_resource(raw_path: str, return '' -# Zip-backed resource loader - -class _ZipLoader: - """Read configs and dicts from a flat OpenCC resource zip archive. - - The zip produced by ``//data:opencc_resources_zip`` stores all ``.json`` - and ``.txt`` files at the root level with no subdirectories. - """ +# opencc-data resource helpers - __slots__ = ('_zf', '_zip_path', '_names') +def _data_resource(filename: str): + return _get_opencc_data().data_path(filename or None) - def __init__(self, zip_path: str) -> None: - self._zip_path = zip_path - self._zf = zipfile.ZipFile(zip_path, 'r') - self._names: frozenset = frozenset(self._zf.namelist()) - def has_txt(self, stem: str) -> bool: - return (stem + '.txt') in self._names +def _config_resource(filename: str): + return _get_opencc_data().config_path(filename or None) - def open_txt(self, stem: str): - """Return a text stream for ``stem.txt``.""" - return io.TextIOWrapper(self._zf.open(stem + '.txt'), encoding='utf-8-sig') - def has_config(self, name: str) -> bool: - fname = name if name.endswith('.json') else name + '.json' - return fname in self._names - - def read_config(self, name: str) -> dict: - fname = name if name.endswith('.json') else name + '.json' - return json.loads(self._zf.read(fname).decode('utf-8')) - - def list_config_names(self) -> list: - return sorted(n for n in self._names if n.endswith('.json')) - - def cache_key(self, stem: str, reverse: bool) -> tuple: - return (self._zip_path, stem, reverse) +def _resource_is_file(resource) -> bool: + try: + return resource.is_file() + except (FileNotFoundError, NotADirectoryError): + return False - def close(self) -> None: - self._zf.close() - def __del__(self) -> None: - try: - self._zf.close() - except Exception: - pass +def _open_text_resource(resource): + return resource.open('r', encoding='utf-8-sig') # Dictionary loading @@ -364,20 +289,18 @@ def __del__(self) -> None: _trie_cache: dict = {} # cache_key -> _Trie -def _find_dict_txt(filename: str, config_dir: str = '', - zip_loader: '_ZipLoader | None' = None): +def _find_dict_txt(filename: str, config_dir: str = ''): """ Resolve a dictionary filename (typically ``Foo.ocd2``) to a - ``(name_or_path, needs_reverse)`` tuple. - - When ``zip_loader`` is given, ``name_or_path`` is the bare stem (e.g. - ``'STPhrases'``) and all I/O goes through the zip. Otherwise it is an - absolute filesystem path. + ``(source, needs_reverse)`` tuple. ``source`` is either a filesystem + path for custom/local dictionaries or a ``Traversable`` from + ``opencc-data`` for built-in dictionaries. Resolution order: 1. Strip the extension (``Foo.ocd2`` → ``Foo``). - 2. Zip lookup or filesystem search (config dir → pkg dir → runfiles → repo). - 3. If the stem ends with ``Rev`` (e.g. ``HKVariantsRev``), look for + 2. Search custom filesystem locations when a config directory is known. + 3. Search ``opencc-data`` packaged dictionaries. + 4. If the stem ends with ``Rev`` (e.g. ``HKVariantsRev``), look for the forward dict (``HKVariants``) and mark it for reversal. """ stem = filename @@ -386,26 +309,12 @@ def _find_dict_txt(filename: str, config_dir: str = '', stem = stem[: -len(ext)] break - if zip_loader is not None: - if zip_loader.has_txt(stem): - return stem, False - if stem.endswith('Rev'): - forward_stem = stem[:-3] - if zip_loader.has_txt(forward_stem): - return forward_stem, True - raise FileNotFoundError(f'Dictionary file not found in zip for: {filename!r}') - if os.path.isabs(filename): path = stem + '.txt' if os.path.isfile(path): return path, False - search_dirs = ( - config_dir, - _pkg_dict_dir, - _runfiles_dict_dir, - _repo_dict_dir, - ) + search_dirs = (config_dir,) for search_dir in search_dirs: if not search_dir: @@ -414,6 +323,10 @@ def _find_dict_txt(filename: str, config_dir: str = '', if os.path.isfile(path): return path, False + resource = _data_resource(stem + '.txt') + if _resource_is_file(resource): + return resource, False + # Try reversed dict: strip the trailing "Rev" and reverse the forward file. if stem.endswith('Rev'): forward_stem = stem[:-3] @@ -423,22 +336,21 @@ def _find_dict_txt(filename: str, config_dir: str = '', path = os.path.join(search_dir, forward_stem + '.txt') if os.path.isfile(path): return path, True + resource = _data_resource(forward_stem + '.txt') + if _resource_is_file(resource): + return resource, True raise FileNotFoundError(f'Dictionary file not found for: {filename!r}') -def _parse_dict_lines(name_or_path: str, - zip_loader: '_ZipLoader | None' = None): +def _parse_dict_lines(source): """ Yield ``(key, candidates)`` tuples for every data line in a txt dict file. - - When ``zip_loader`` is given, ``name_or_path`` is a bare stem read from - the zip; otherwise it is a filesystem path. """ - if zip_loader is not None: - f = zip_loader.open_txt(name_or_path) + if isinstance(source, str): + f = open(source, encoding='utf-8-sig') else: - f = open(name_or_path, encoding='utf-8-sig') + f = _open_text_resource(source) with f: for line in f: line = line.strip() @@ -450,8 +362,7 @@ def _parse_dict_lines(name_or_path: str, yield key, candidates -def _load_trie(name_or_path: str, reverse: bool, - zip_loader: '_ZipLoader | None' = None) -> _Trie: +def _load_trie(source, reverse: bool) -> _Trie: """ Load a ``keyvalue1 value2 ...`` txt file into a :class:`_Trie`. @@ -461,10 +372,8 @@ def _load_trie(name_or_path: str, reverse: bool, encountered (in file order) wins, matching the behaviour of the C++ dict-reversal script. """ - if zip_loader is not None: - cache_key = zip_loader.cache_key(name_or_path, reverse) - else: - cache_key = (os.path.abspath(name_or_path), reverse) + cache_source = os.path.abspath(source) if isinstance(source, str) else str(source) + cache_key = (cache_source, reverse) cached = _trie_cache.get(cache_key) if cached is not None: @@ -473,12 +382,12 @@ def _load_trie(name_or_path: str, reverse: bool, trie = _Trie() if not reverse: - for key, candidates in _parse_dict_lines(name_or_path, zip_loader): + for key, candidates in _parse_dict_lines(source): trie.add(key, candidates[0]) else: # Reverse: each candidate value -> original key (first writer wins). reversed_mapping: dict = {} - for key, candidates in _parse_dict_lines(name_or_path, zip_loader): + for key, candidates in _parse_dict_lines(source): for candidate in candidates: if candidate not in reversed_mapping: reversed_mapping[candidate] = key @@ -489,29 +398,15 @@ def _load_trie(name_or_path: str, reverse: bool, return trie -def _get_trie(filename: str, config_dir: str = '', - zip_loader: '_ZipLoader | None' = None) -> _Trie: - """Resolve ``filename`` and return the loaded trie. - - Tries the filesystem first (catches both source and Bazel runfile - locations), then falls back to the resource zip for generated files - that are not present in the source tree. - """ - try: - fs_path, needs_reverse = _find_dict_txt(filename, config_dir, zip_loader=None) - return _load_trie(fs_path, needs_reverse, zip_loader=None) - except FileNotFoundError: - pass - if zip_loader is not None: - stem, needs_reverse = _find_dict_txt(filename, config_dir, zip_loader=zip_loader) - return _load_trie(stem, needs_reverse, zip_loader=zip_loader) - raise FileNotFoundError(f'Dictionary file not found for: {filename!r}') +def _get_trie(filename: str, config_dir: str = '') -> _Trie: + """Resolve ``filename`` and return the loaded trie.""" + source, needs_reverse = _find_dict_txt(filename, config_dir) + return _load_trie(source, needs_reverse) # Config parsing def _parse_dict_node(d: dict, config_dir: str = '', - zip_loader: '_ZipLoader | None' = None, include_tofu_risk_dicts: bool = True): """ Recursively parse a JSON dict-config node. @@ -527,7 +422,7 @@ def _parse_dict_node(d: dict, config_dir: str = '', policy = d.get('match_policy', 'short_circuit') matchers = [ m for sub in d['dicts'] - if (m := _parse_dict_node(sub, config_dir, zip_loader, include_tofu_risk_dicts)) is not None + if (m := _parse_dict_node(sub, config_dir, include_tofu_risk_dicts)) is not None ] if not matchers: return None @@ -537,7 +432,7 @@ def _parse_dict_node(d: dict, config_dir: str = '', return None if dtype in ('ocd2', 'ocd', 'txt', 'text'): - return _get_trie(d['file'], config_dir, zip_loader) + return _get_trie(d['file'], config_dir) raise ValueError(f'Unknown dict type: {dtype!r}') @@ -603,60 +498,22 @@ def _segment(text: str, matcher: _GroupMatcher) -> list: def _find_config(config_name: str) -> str: """ Locate a config JSON file by name (with or without ``.json`` suffix). - - Searches the bundled package config directory first, then the source-tree - ``data/config`` directory as a fallback. """ filename = config_name if config_name.endswith('.json') else config_name + '.json' - for search_dir in ( - _pkg_config_dir, - _runfiles_config_dir, - _repo_config_dir, - _runfiles_jieba_config_dir, - _repo_jieba_config_dir): - if not search_dir: - continue - path = os.path.join(search_dir, filename) - if os.path.isfile(path): - return path + resource = _config_resource(filename) + if _resource_is_file(resource): + return resource raise FileNotFoundError(f'Config not found: {config_name!r}') -def _default_zip_loader() -> '_ZipLoader | None': - """Return a _ZipLoader for the bundled resource zip, or None.""" - env_zip = os.environ.get('OPENCC_RESOURCE_ZIP', '') - if env_zip and os.path.isfile(env_zip): - return _ZipLoader(env_zip) - if os.path.isfile(_pkg_resource_zip): - return _ZipLoader(_pkg_resource_zip) - return None - - def _config_seg_type(cfg: dict) -> str: return cfg.get('segmentation', {}).get('type', '') -def _config_uses_subdir_dicts(cfg: dict) -> bool: - """Return True if the config references any dict file under a subdirectory.""" - def _check(value) -> bool: - if isinstance(value, dict): - for key, child in value.items(): - if key == 'file' and isinstance(child, str) and '/' in child: - return True - if _check(child): - return True - elif isinstance(value, list): - for item in value: - if _check(item): - return True - return False - return _check(cfg) - - def _is_supported_cfg(cfg: dict, config_path: str = '') -> bool: """Return True if the pure Python backend can use this parsed config.""" seg_type = _config_seg_type(cfg) - if seg_type == 'mmseg': + if seg_type in ('', 'mmseg'): return True if seg_type == 'jieba': if not _jieba_available(): @@ -671,75 +528,23 @@ def _is_supported_cfg(cfg: dict, config_path: str = '') -> bool: return False -def list_configs(resource_zip: 'str | None' = None) -> list: +def list_configs() -> list: """ Return a sorted list of ``.json`` config filenames available to the - pure Python converter. Jieba configs are listed only when the optional - ``jieba`` package and dictionary data are available. - - When ``resource_zip`` is given (or the package ships a bundled - ``opencc-resources.zip``), configs are enumerated from the zip. + pure Python converter. """ - zip_loader: '_ZipLoader | None' = None - if resource_zip is not None: - zip_loader = _ZipLoader(resource_zip) - elif resource_zip is None: - zip_loader = _default_zip_loader() - - if zip_loader is not None: - try: - configs = [] - seen: set = set() - for name in zip_loader.list_config_names(): - try: - cfg = zip_loader.read_config(name) - if _is_supported_cfg(cfg): - configs.append(name) - seen.add(name) - except Exception: - pass - # Jieba configs are not bundled in the zip; scan filesystem. - for search_dir in (_runfiles_jieba_config_dir, _repo_jieba_config_dir): - if not search_dir or not os.path.isdir(search_dir): - continue - for fname in sorted(os.listdir(search_dir)): - if not fname.endswith('.json') or fname in seen: - continue - path = os.path.join(search_dir, fname) - try: - with open(path, encoding='utf-8') as f: - cfg = json.load(f) - if _is_supported_cfg(cfg, path) and not _config_uses_subdir_dicts(cfg): - configs.append(fname) - seen.add(fname) - except Exception: - pass - return sorted(configs) - finally: - zip_loader.close() - configs = [] - seen: set = set() - for search_dir in ( - _pkg_config_dir, - _runfiles_config_dir, - _repo_config_dir, - _runfiles_jieba_config_dir, - _repo_jieba_config_dir): - if not search_dir or not os.path.isdir(search_dir): + config_dir = _config_resource('') + for resource in config_dir.iterdir(): + name = resource.name + if not name.endswith('.json'): continue - for fname in os.listdir(search_dir): - if not fname.endswith('.json') or fname in seen: - continue - path = os.path.join(search_dir, fname) - try: - with open(path, encoding='utf-8') as f: - cfg = json.load(f) - if _is_supported_cfg(cfg, path): - configs.append(fname) - seen.add(fname) - except Exception: - pass + try: + cfg = json.loads(resource.read_text(encoding='utf-8')) + if _is_supported_cfg(cfg): + configs.append(name) + except Exception: + pass return sorted(configs) @@ -762,19 +567,17 @@ class OpenCC: def __init__(self, config: str = 't2s', resource_zip: 'str | None' = None, include_tofu_risk_dictionaries: bool = True) -> None: + if resource_zip is not None: + raise ValueError( + 'resource_zip is no longer supported; install the opencc-data ' + 'package to provide built-in configs and dictionaries.' + ) + config_name = config[:-5] if config.endswith('.json') else config suffixed_config = config if config.endswith('.json') else f'{config}.json' - # Determine the active zip loader. - if resource_zip is not None: - zip_loader: '_ZipLoader | None' = _ZipLoader(resource_zip) - else: - zip_loader = _default_zip_loader() - self._zip_loader = zip_loader - # Load the config JSON. - # Filesystem paths take priority over the zip so that custom configs - # and jieba plugin configs (not bundled in the zip) work correctly. + # Filesystem paths take priority so that custom configs work correctly. config_dir = '' if os.path.isfile(config): with open(config, encoding='utf-8') as f: @@ -784,23 +587,17 @@ def __init__(self, config: str = 't2s', with open(suffixed_config, encoding='utf-8') as f: cfg = json.load(f) config_dir = os.path.dirname(os.path.abspath(suffixed_config)) - elif zip_loader is not None and zip_loader.has_config(config_name): - cfg = zip_loader.read_config(config_name) else: - config_path = _find_config(config_name) - with open(config_path, encoding='utf-8') as f: - cfg = json.load(f) - config_dir = os.path.dirname(config_path) + config_resource = _find_config(config_name) + cfg = json.loads(config_resource.read_text(encoding='utf-8')) self.config = suffixed_config segmentation = cfg.get('segmentation', {}) seg_type = segmentation.get('type', '') include_tofu = include_tofu_risk_dictionaries - # Dict loading always passes zip_loader as a fallback for generated files - # that are not present in the source tree (e.g. STPhrases_Generated…, TSCharactersExt). if seg_type == 'mmseg': - self._segmenter = _parse_dict_node(segmentation['dict'], config_dir, zip_loader, include_tofu) + self._segmenter = _parse_dict_node(segmentation['dict'], config_dir, include_tofu) elif seg_type == 'jieba': self._segmenter = _JiebaSegmenter( segmentation.get('resources', {}), @@ -816,12 +613,12 @@ def __init__(self, config: str = 't2s', self._normalization: list = [ m for step in cfg.get('normalization', []) - if (m := _parse_dict_node(step['dict'], config_dir, zip_loader, include_tofu)) is not None + if (m := _parse_dict_node(step['dict'], config_dir, include_tofu)) is not None ] self._chain: list = [ m for step in cfg['conversion_chain'] - if (m := _parse_dict_node(step['dict'], config_dir, zip_loader, include_tofu)) is not None + if (m := _parse_dict_node(step['dict'], config_dir, include_tofu)) is not None ] def convert(self, text: str) -> str: diff --git a/python-pure/setup.py b/python-pure/setup.py index 93008f840..c35c9eb6e 100644 --- a/python-pure/setup.py +++ b/python-pure/setup.py @@ -1,86 +1,11 @@ -import json import os -import re -import shutil -import subprocess import setuptools -import setuptools.command.build_py _this_dir = os.path.dirname(os.path.abspath(__file__)) -_repo_root = os.path.abspath(os.path.join(_this_dir, '..')) -_author_file = os.path.join(_repo_root, 'AUTHORS') -_readme_file = os.path.join(_repo_root, 'README.md') -_fallback_version = '1.3.1' - - -def _get_version_from_git(): - try: - raw = subprocess.check_output( - ['git', 'describe', '--tags', '--long', '--always'], - cwd=_repo_root, - stderr=subprocess.DEVNULL, - ).decode('utf-8').strip() - except (OSError, subprocess.CalledProcessError): - return '' - - dirty = '' - for diff_cmd in (['git', 'diff', '--quiet'], ['git', 'diff', '--cached', '--quiet']): - try: - subprocess.check_call( - diff_cmd, - cwd=_repo_root, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - except (OSError, subprocess.CalledProcessError): - dirty = '.dirty' - break - - release_match = re.match(r'^(?:v|ver\.)(\d+\.\d+\.\d+)-0-g[0-9a-f]+$', raw) - if release_match: - return '{}{}'.format(release_match.group(1), dirty) - - dev_match = re.match(r'^(?:v|ver\.)(\d+\.\d+\.\d+)-(\d+)-g([0-9a-f]+)$', raw) - if dev_match: - return '{}.dev{}+g{}{}'.format( - dev_match.group(1), - dev_match.group(2), - dev_match.group(3), - dirty, - ) - - return '{}+g{}{}'.format(_fallback_version, raw, dirty) - - -def get_version_info(): - env_version = os.environ.get('VERSION', '') - if env_version: - return env_version.lstrip('v') - git_version = _get_version_from_git() - if git_version: - return git_version - return _fallback_version - - -def get_author_info(): - if not os.path.isfile(_author_file): - return 'BYVoid', 'byvoid@byvoid.com' - - authors = [] - emails = [] - author_pattern = re.compile(r'(.+) <(.+)>') - with open(_author_file, 'rb') as f: - for line in f: - match = author_pattern.search(line.decode('utf-8')) - if not match: - continue - authors.append(match.group(1)) - emails.append(match.group(2)) - if not authors: - return 'BYVoid', 'byvoid@byvoid.com' - return ', '.join(authors), ', '.join(emails) +_readme_file = os.path.join(_this_dir, 'README.md') +OPENCC_DATA_VERSION = '1.3.2.dev20260628' def get_long_description(): @@ -90,179 +15,25 @@ def get_long_description(): return f.read() -def _load_trie_from_txt(path): - """Build a minimal {key: first_value} prefix trie from a .txt dict file.""" - trie = {} - with open(path, encoding='utf-8-sig') as f: - for line in f: - line = line.strip() - if not line or line.startswith('#') or '\t' not in line: - continue - key, rest = line.split('\t', 1) - value = rest.split()[0] - node = trie - for i, ch in enumerate(key): - if ch not in node: - node[ch] = {} - node = node[ch] - node.setdefault('__v__', value) - return trie - - -def _trie_match(trie, text, pos): - """Longest prefix match; returns (length, value) or (0, None).""" - node = trie - best_len, best_val = 0, None - i = pos - while i < len(text): - node = node.get(text[i]) - if node is None: - break - i += 1 - if '__v__' in node: - best_len, best_val = i - pos, node['__v__'] - return best_len, best_val - - -def _t2s_convert(text, tries): - """Convert traditional text to simplified using an ordered list of tries - (short-circuit: first trie that matches at a position wins).""" - result = [] - pos = 0 - while pos < len(text): - matched = False - for trie in tries: - length, value = _trie_match(trie, text, pos) - if length > 0: - result.append(value) - pos += length - matched = True - break - if not matched: - result.append(text[pos]) - pos += 1 - return ''.join(result) - - -def _generate_st_phrases_from_regional(src_dict, dst_dict): - """Generate STPhrases_GeneratedFromRegionalPhrases.txt in dst_dict. - - Mirrors the logic of data/scripts/generate_st_phrases_from_regional_phrases.py: - convert TWPhrases/HKPhrases keys via t2s and emit keys with len >= 3. - """ - ts_phrases_path = os.path.join(src_dict, 'TSPhrases.txt') - ts_chars_path = os.path.join(src_dict, 'TSCharacters.txt') - if not os.path.isfile(ts_phrases_path) or not os.path.isfile(ts_chars_path): - return - - tries = [_load_trie_from_txt(ts_phrases_path), - _load_trie_from_txt(ts_chars_path)] - - collisions = {} # simplified_key -> first_traditional_key - for fname in ('TWPhrases.txt', 'HKPhrases.txt'): - path = os.path.join(src_dict, fname) - if not os.path.isfile(path): - continue - with open(path, encoding='utf-8-sig') as f: - for line in f: - line = line.strip() - if not line or line.startswith('#') or '\t' not in line: - continue - original_key = line.split('\t', 1)[0] - simplified_key = _t2s_convert(original_key, tries) - if len(simplified_key) < 3: - continue - collisions.setdefault(simplified_key, original_key) - - out_path = os.path.join(dst_dict, 'STPhrases_GeneratedFromRegionalPhrases.txt') - with open(out_path, 'w', encoding='utf-8', newline='\n') as f: - f.write('# Open Chinese Convert (OpenCC) Dictionary\n') - f.write('# File: STPhrases_GeneratedFromRegionalPhrases.txt\n') - f.write('# Format: key\tvalue(s) (values separated by spaces)\n') - f.write('# License: Apache-2.0 (see LICENSE)\n') - f.write('# Source: generated from TWPhrases.txt, HKPhrases.txt keys via t2s\n') - f.write('# Used in configs: s2t.json, s2hk.json, s2tw.json\n') - f.write('#\n') - f.write('\n') - for key in sorted(collisions): - f.write('{}\t{}\n'.format(key, collisions[key])) - - -class BuildPyCommand(setuptools.command.build_py.build_py, object): - """Bundle OpenCC's JSON configs and text dictionaries into the package.""" - - def run(self): - super(BuildPyCommand, self).run() - self._copy_opencc_data() - - def _copy_opencc_data(self): - src_config = os.path.join(_repo_root, 'data', 'config') - src_dict = os.path.join(_repo_root, 'data', 'dictionary') - src_jieba_config = os.path.join(_repo_root, 'plugins', 'jieba', 'data', 'config') - src_jieba_dict = os.path.join(_repo_root, 'plugins', 'jieba', 'deps', 'cppjieba', 'dict') - dst_base = os.path.join(self.build_lib, 'opencc') - dst_config = os.path.join(dst_base, 'config') - dst_dict = os.path.join(dst_base, 'dictionary') - dst_jieba_dict = os.path.join(dst_base, 'jieba_dict') - - os.makedirs(dst_config, exist_ok=True) - for fname in os.listdir(src_config): - if not fname.endswith('.json'): - continue - src_config_path = os.path.join(src_config, fname) - with open(src_config_path, encoding='utf-8') as f: - config = json.load(f) - if config.get('segmentation', {}).get('type') != 'mmseg': - continue - shutil.copy2(src_config_path, os.path.join(dst_config, fname)) - if os.path.isdir(src_jieba_config): - for fname in os.listdir(src_jieba_config): - if fname.endswith('.json'): - shutil.copy2( - os.path.join(src_jieba_config, fname), - os.path.join(dst_config, fname), - ) - - os.makedirs(dst_dict, exist_ok=True) - for fname in os.listdir(src_dict): - if fname.endswith('.txt'): - shutil.copy2( - os.path.join(src_dict, fname), - os.path.join(dst_dict, fname), - ) - - _generate_st_phrases_from_regional(src_dict, dst_dict) - - os.makedirs(dst_jieba_dict, exist_ok=True) - for fname in ('jieba.dict.utf8', 'user.dict.utf8'): - src_path = os.path.join(src_jieba_dict, fname) - if os.path.isfile(src_path): - shutil.copy2(src_path, os.path.join(dst_jieba_dict, fname)) - - -version_info = get_version_info() -author_info = get_author_info() - setuptools.setup( name='opencc-py', - version=version_info, - author=author_info[0], - author_email=author_info[1], + version=OPENCC_DATA_VERSION, + author='OpenCC contributors', description='Conversion between Traditional and Simplified Chinese (pure Python)', long_description=get_long_description(), long_description_content_type='text/markdown', url='https://github.com/BYVoid/OpenCC', + license='Apache-2.0', + license_files=['LICENSE'], packages=['opencc'], package_dir={'opencc': 'opencc'}, package_data={ - 'opencc': ['py.typed', 'config/*.json', 'dictionary/*.txt', 'jieba_dict/*.utf8'], - }, - cmdclass={ - 'build_py': BuildPyCommand, + 'opencc': ['py.typed'], }, + install_requires=[f'opencc-data=={OPENCC_DATA_VERSION}'], - python_requires='>=3.8', + python_requires='>=3.9', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', @@ -271,7 +42,6 @@ def _copy_opencc_data(self): 'Natural Language :: Chinese (Traditional)', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'License :: OSI Approved :: Apache Software License', 'Topic :: Scientific/Engineering', 'Topic :: Software Development', 'Topic :: Software Development :: Libraries', @@ -280,6 +50,5 @@ def _copy_opencc_data(self): 'Topic :: Text Processing :: Linguistic', 'Typing :: Typed', ], - license='Apache License 2.0', keywords=['opencc', 'convert', 'chinese'], ) diff --git a/python-pure/tests/requirements_lock.txt b/python-pure/tests/requirements_lock.txt index 47e5801dd..b2e1f57d8 100644 --- a/python-pure/tests/requirements_lock.txt +++ b/python-pure/tests/requirements_lock.txt @@ -1,5 +1,6 @@ exceptiongroup==1.2.2 iniconfig==2.0.0 +opencc-data==1.3.2.dev20260628 packaging==24.1 pluggy==1.5.0 pygments==2.20.0 diff --git a/python-pure/tests/test_opencc.py b/python-pure/tests/test_opencc.py index 68c2ee0a3..c8d4f2703 100644 --- a/python-pure/tests/test_opencc.py +++ b/python-pure/tests/test_opencc.py @@ -1,25 +1,13 @@ import json -import importlib.util import os import pytest import sys +import opencc_data + _this_dir = os.path.dirname(os.path.abspath(__file__)) _opencc_rootdir = os.path.abspath(os.path.join(_this_dir, '..', '..')) -_testcases_path = os.path.join(_opencc_rootdir, 'test', 'testcases', 'testcases.json') _config_test_path = os.path.join(_opencc_rootdir, 'test', 'config_test', 'config_test.json') -_jieba_testcases_path = os.path.join( - _opencc_rootdir, - 'plugins', - 'jieba', - 'tests', - 'data', - 'jieba_comparison_testcases.json', -) - - -def _has_jieba(): - return importlib.util.find_spec('jieba') is not None def test_import(): @@ -34,41 +22,22 @@ def test_init_delete_converter(): del converter -def test_jieba_configs_are_exposed_only_when_optional_dependency_is_available(): +def test_configs_are_loaded_from_opencc_data(): import opencc - if _has_jieba(): - assert any('jieba' in config for config in opencc.CONFIGS) - else: - assert all('jieba' not in config for config in opencc.CONFIGS) + expected = sorted( + resource.name + for resource in opencc_data.config_path().iterdir() + if resource.name.endswith('.json') + ) + assert opencc.CONFIGS == expected -def test_jieba_config_requires_optional_dependency_when_missing(): - if _has_jieba(): - pytest.skip('jieba is installed') +def test_jieba_configs_are_not_exposed_by_opencc_data(): import opencc - with pytest.raises(ImportError, match='pip install jieba'): - opencc.OpenCC('s2t_jieba') - - -@pytest.mark.skipif(not _has_jieba(), reason='jieba optional dependency is not installed') -def test_jieba_configs_match_plugin_comparison_cases(): - import opencc - - with open(_jieba_testcases_path, 'r', encoding='utf-8') as f: - parsed = json.load(f) - - for case in parsed['cases']: - for cfg, ans in case.get('expected', {}).items(): - if not cfg.endswith('_jieba'): - continue - # Jieba comparison testcases record CLI golden outputs, which skip - # tofu-risk dictionaries by default (same as --include-tofu-risk-dictionaries=false). - converter = opencc.OpenCC(f'{cfg}.json', include_tofu_risk_dictionaries=False) - assert converter.convert(case['input']) == ans, \ - 'Failed to convert {} for {} -> {}'.format(cfg, case['input'], ans) + assert all('jieba' not in config for config in opencc.CONFIGS) def test_custom_config_resolves_local_dictionaries(): @@ -129,7 +98,7 @@ def test_dictionary_parser_strips_utf8_bom(tmp_path): def test_conversion(): import opencc - with open(_testcases_path, 'r', encoding='utf-8') as f: + with opencc_data.test_data_path('testcases.json').open('r', encoding='utf-8') as f: parsed = json.load(f) for case in parsed.get('cases', []): From 2e5a22a6c98266a5c95c582c6e83266fe07cce55 Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Sun, 28 Jun 2026 07:56:21 -0700 Subject: [PATCH 10/13] Combine pure Python package descriptions Build PKG-INFO and wheel metadata from both English and Traditional Chinese README files while stripping language-switch links and Markdown URLs. Include both README files in the source distribution manifest. Detailed Changes: - **Package Metadata**: - Compose long_description from README.md and README.zh-TW.md. - Remove README language-switch links and Markdown link targets from generated metadata. - **Source Distribution**: - Add MANIFEST.in so README.md, README.zh-TW.md, and LICENSE are explicitly included. --- python-pure/MANIFEST.in | 3 +++ python-pure/setup.py | 28 +++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 python-pure/MANIFEST.in diff --git a/python-pure/MANIFEST.in b/python-pure/MANIFEST.in new file mode 100644 index 000000000..f1be660e5 --- /dev/null +++ b/python-pure/MANIFEST.in @@ -0,0 +1,3 @@ +include README.md +include README.zh-TW.md +include LICENSE diff --git a/python-pure/setup.py b/python-pure/setup.py index c35c9eb6e..a8d826cc5 100644 --- a/python-pure/setup.py +++ b/python-pure/setup.py @@ -1,18 +1,36 @@ import os +import re import setuptools _this_dir = os.path.dirname(os.path.abspath(__file__)) -_readme_file = os.path.join(_this_dir, 'README.md') +_readme_files = ( + os.path.join(_this_dir, 'README.md'), + os.path.join(_this_dir, 'README.zh-TW.md'), +) OPENCC_DATA_VERSION = '1.3.2.dev20260628' +def _read_readme_for_pkg_info(path): + with open(path, encoding='utf-8') as f: + content = f.read() + content = re.sub( + r'^\[(?:English version|繁體中文版)\]\([^)]+\)\n\n', + '', + content, + flags=re.MULTILINE, + ) + return re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', content).strip() + + def get_long_description(): - if not os.path.isfile(_readme_file): - return '' - with open(_readme_file, encoding='utf-8') as f: - return f.read() + parts = [ + _read_readme_for_pkg_info(path) + for path in _readme_files + if os.path.isfile(path) + ] + return '\n\n---\n\n'.join(parts) setuptools.setup( From d86605424af1538023ee12992a733e5298efca32 Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Sun, 28 Jun 2026 08:03:16 -0700 Subject: [PATCH 11/13] Document opencc-py installation Update the English and Traditional Chinese pure Python READMEs to name the published package explicitly and show the user-facing pip install command. Detailed Changes: - **Installation Docs**: - State that the PyPI package name is opencc-py. - Add python -m pip install opencc-py to both README files. - Keep local and editable install instructions for development. --- python-pure/README.md | 10 ++++++++-- python-pure/README.zh-TW.md | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/python-pure/README.md b/python-pure/README.md index 435b5fdec..5b54ae293 100644 --- a/python-pure/README.md +++ b/python-pure/README.md @@ -32,7 +32,13 @@ looks for the corresponding `CustomPhrases.txt` next to the config file. ## Installation -Install from this directory with pip: +The PyPI package name is `opencc-py`. Users can install it with pip: + +```bash +python -m pip install opencc-py +``` + +For local development from this directory: ```bash python -m pip install . @@ -42,7 +48,7 @@ The package version matches its `opencc-data` version and declares the matching data package as an exact install dependency, so pip installs the compatible data package automatically. -For editable development: +Or use editable development mode: ```bash python -m pip install -e . diff --git a/python-pure/README.zh-TW.md b/python-pure/README.zh-TW.md index a4a27a669..ac8195d75 100644 --- a/python-pure/README.zh-TW.md +++ b/python-pure/README.zh-TW.md @@ -30,7 +30,13 @@ print(converter.convert("汉字")) # 漢字 ## 安裝 -在此目錄中使用 pip 安裝： +PyPI package 名稱是 `opencc-py`。使用者可以透過 pip 安裝： + +```bash +python -m pip install opencc-py +``` + +從此目錄進行本地開發安裝： ```bash python -m pip install . @@ -39,7 +45,7 @@ python -m pip install . 此 package 的版本會與 `opencc-data` 版本一致，並將相同版本的資料 package 宣告為精確安裝依賴，因此 pip 會自動安裝相容的資料 package。 -開發時可使用 editable 安裝： +也可以使用 editable development mode： ```bash python -m pip install -e . From 23b943e6d255a45e18df40223f9c5aa48a0cbef9 Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Sun, 28 Jun 2026 08:04:55 -0700 Subject: [PATCH 12/13] Document official Python opencc package Link the pure Python README comparison section to the official opencc PyPI package and state that it is the official Python implementation. Detailed Changes: - **Documentation**: - Add the official opencc PyPI package link to the English README. - Add the official opencc PyPI package link to the Traditional Chinese README. - Clarify that opencc, not opencc-py, is the official Python implementation. --- python-pure/README.md | 5 +++-- python-pure/README.zh-TW.md | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/python-pure/README.md b/python-pure/README.md index 5b54ae293..255d3b3d4 100644 --- a/python-pure/README.md +++ b/python-pure/README.md @@ -1,4 +1,4 @@ -# OpenCC Pure Python +# opencc-py (OpenCC Pure Python Implementation) [繁體中文版](README.zh-TW.md) @@ -88,7 +88,8 @@ The tests verify: This package intentionally implements only the pieces needed for pure Python text conversion. Compared with the official C++ library and command-line tools, -it omits several lower-level details: +it omits several lower-level details. The official Python implementation is the +[`opencc` PyPI package](https://pypi.org/project/opencc/). - binary dictionary loading for `.ocd2`/`.ocd`; built-in dictionaries are read from `.txt` data supplied by `opencc-data` diff --git a/python-pure/README.zh-TW.md b/python-pure/README.zh-TW.md index ac8195d75..3614280f4 100644 --- a/python-pure/README.zh-TW.md +++ b/python-pure/README.zh-TW.md @@ -1,4 +1,4 @@ -# OpenCC Pure Python +# opencc-py (OpenCC 純 Python 實作) [English version](README.md) @@ -83,7 +83,8 @@ PYTHONPATH=python-pure python -m pytest python-pure/tests ## 與官方實作的差異 此 package 刻意只實作純 Python 文字轉換所需的部分。相較於官方 C++ library -與 command-line tools，它省略了幾個較底層的實作細節： +與 command-line tools，它省略了幾個較底層的實作細節。官方 Python 實作是 PyPI +上的 [`opencc` package](https://pypi.org/project/opencc/)。 - `.ocd2` / `.ocd` 二進位 dictionary 載入；內建 dictionary 會讀取 `opencc-data` 提供的 `.txt` 資料 From f0970f78491535a593d83a387e91c15d6a1c12de Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Sun, 28 Jun 2026 08:14:18 -0700 Subject: [PATCH 13/13] Add pure Python PyPI release workflow Add a manual workflow for validating and publishing the opencc-py pure Python package. The workflow defaults to dry-run validation and publishes through PyPI Trusted Publishing only when dry_run is disabled. Detailed Changes: - **Release Workflow**: - Install pure Python build and test dependencies. - Run the python-pure pytest suite before packaging. - Build sdist and wheel artifacts from python-pure. - Run twine check and upload built distributions as GitHub artifacts. - Publish to PyPI via pypa/gh-action-pypi-publish when dry_run is false. --- .github/workflows/release-pypi-pure.yml | 52 +++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/release-pypi-pure.yml diff --git a/.github/workflows/release-pypi-pure.yml b/.github/workflows/release-pypi-pure.yml new file mode 100644 index 000000000..a1e33be34 --- /dev/null +++ b/.github/workflows/release-pypi-pure.yml @@ -0,0 +1,52 @@ +name: Build and upload pure Python package to PyPI + +on: + workflow_dispatch: + inputs: + dry_run: + description: Build and validate package without publishing to PyPI + required: true + default: true + type: boolean + +jobs: + release-pypi-pure: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install build and test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build twine + python -m pip install -r python-pure/tests/requirements_lock.txt + + - name: Test pure Python package + run: PYTHONPATH=python-pure python -m pytest -o cache_dir=/tmp/opencc-pure-pytest-cache python-pure/tests + + - name: Build pure Python distributions + run: python -m build --sdist --wheel --outdir dist python-pure + + - name: Check distributions + run: python -m twine check dist/* + + - name: Upload distribution artifacts + uses: actions/upload-artifact@v7 + with: + name: opencc-py-dist + path: dist/* + + - name: Publish to PyPI + if: ${{ !inputs.dry_run }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/