diff --git a/.github/workflows/release-pypi-pure.yml b/.github/workflows/release-pypi-pure.yml new file mode 100644 index 000000000..a1e33be34 --- /dev/null +++ b/.github/workflows/release-pypi-pure.yml @@ -0,0 +1,52 @@ +name: Build and upload pure Python package to PyPI + +on: + workflow_dispatch: + inputs: + dry_run: + description: Build and validate package without publishing to PyPI + required: true + default: true + type: boolean + +jobs: + release-pypi-pure: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install build and test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build twine + python -m pip install -r python-pure/tests/requirements_lock.txt + + - name: Test pure Python package + run: PYTHONPATH=python-pure python -m pytest -o cache_dir=/tmp/opencc-pure-pytest-cache python-pure/tests + + - name: Build pure Python distributions + run: python -m build --sdist --wheel --outdir dist python-pure + + - name: Check distributions + run: python -m twine check dist/* + + - name: Upload distribution artifacts + uses: actions/upload-artifact@v7 + with: + name: opencc-py-dist + path: dist/* + + - name: Publish to PyPI + if: ${{ !inputs.dry_run }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ diff --git a/README.md b/README.md index a15328189..2c53fee52 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,17 @@ See [demo.js](https://github.com/BYVoid/OpenCC/blob/master/node/demo.js) and [ts `pip install opencc` (Windows, Linux, macOS) +The Python package uses the pure Python implementation and supports the built-in +mmseg-based configs. Jieba plugin configs are not supported by the Python +package. + +The pure Python implementation loads text dictionaries. Built-in configs keep +their standard `.ocd2` dictionary references, and the Python loader resolves +those references to bundled same-stem `.txt` dictionaries when possible. Custom +configs that reference `.ocd` or `.ocd2` dictionaries must also provide matching +text dictionary files; binary dictionary files are not parsed by the Python +package. + ```python import opencc converter = opencc.OpenCC('s2t.json') diff --git a/data/BUILD.bazel b/data/BUILD.bazel index 912505dd1..e654e6354 100644 --- a/data/BUILD.bazel +++ b/data/BUILD.bazel @@ -16,6 +16,17 @@ opencc_resources_zip( out = "opencc-resources.zip", ) +opencc_resources_zip( + name = "opencc_py_resources_zip", + configs = ["//data/config:config"], + dicts = [ + "//data/dictionary:text_dictionaries", + "//data/dictionary:CJK_Compatibility_Ideographs.txt", + ], + exclude_dict_subdirs = True, + out = "opencc-py-resources.zip", +) + opencc_resources_ocd2_zip( name = "opencc_resources_ocd2_zip", src = ":opencc_resources_zip", diff --git a/data/scripts/opencc_resources_zip.bzl b/data/scripts/opencc_resources_zip.bzl index dc67c10d8..4fb218c33 100644 --- a/data/scripts/opencc_resources_zip.bzl +++ b/data/scripts/opencc_resources_zip.bzl @@ -6,6 +6,8 @@ def _opencc_resources_zip_impl(ctx): args.add("--volatile-status", ctx.version_file) args.add_all("--configs", ctx.files.configs) args.add_all("--dicts", ctx.files.dicts) + if ctx.attr.exclude_dict_subdirs: + args.add("--exclude-dict-subdirs") ctx.actions.run( executable = ctx.executable.tool, @@ -25,6 +27,7 @@ opencc_resources_zip = rule( attrs = { "configs": attr.label_list(allow_files = True, mandatory = True), "dicts": attr.label_list(allow_files = True, mandatory = True), + "exclude_dict_subdirs": attr.bool(default = False), "source_url": attr.string(default = "https://github.com/BYVoid/OpenCC"), "tool": attr.label( default = Label("//data/scripts:opencc_resources_zip"), diff --git a/data/scripts/opencc_resources_zip.py b/data/scripts/opencc_resources_zip.py index 040193ba4..a7ad01094 100644 --- a/data/scripts/opencc_resources_zip.py +++ b/data/scripts/opencc_resources_zip.py @@ -24,6 +24,12 @@ def parse_args(): parser.add_argument("--source-url", default=DEFAULT_SOURCE_URL) parser.add_argument("--stable-status") parser.add_argument("--volatile-status") + parser.add_argument( + "--exclude-dict-subdirs", + action="store_true", + default=False, + help="Skip dict files stored under a subdirectory (e.g. cngov/).", + ) return parser.parse_args() @@ -146,6 +152,21 @@ def convert_dict_references(value): return value +def config_references_subdir_dict(value): + """Return True if the config JSON references any dict file under a subdirectory.""" + if isinstance(value, dict): + for key, child in value.items(): + if key == "file" and isinstance(child, str) and "/" in child: + return True + if config_references_subdir_dict(child): + return True + elif isinstance(value, list): + for item in value: + if config_references_subdir_dict(item): + return True + return False + + def read_text_config(path): with open(path, encoding="utf-8") as file: config = json.load(file) @@ -172,6 +193,8 @@ def write_entry(archive, name, content, date_time): def add_entry(archive, entries, name, content, date_time): + if name in entries: + return write_entry(archive, name, content, date_time) entries[name] = { "sha256": hashlib.sha256(content.encode("utf-8")).hexdigest(), @@ -179,6 +202,21 @@ def add_entry(archive, entries, name, content, date_time): } +def dict_zip_name(path): + """Return the name for a dict file inside the zip archive. + + Preserves subdirectory components relative to the nearest 'dictionary' + ancestor so that files in subdirectories (e.g. cngov/STCharacters.txt) + are stored with their subpath and do not collide with top-level files + of the same basename. + """ + parts = path.replace("\\", "/").split("/") + for i in range(len(parts) - 2, -1, -1): + if parts[i] == "dictionary": + return "/".join(parts[i + 1 :]) + return os.path.basename(path) + + def main(): args = parse_args() output_dir = os.path.dirname(args.output) @@ -188,6 +226,11 @@ def main(): with zipfile.ZipFile(args.output, "w") as archive: entries = {} for path in sorted(args.configs, key=os.path.basename): + if args.exclude_dict_subdirs: + with open(path, encoding="utf-8") as f: + raw_cfg = json.load(f) + if config_references_subdir_dict(raw_cfg): + continue add_entry( archive, entries, @@ -195,11 +238,14 @@ def main(): read_text_config(path), date_time, ) - for path in sorted(args.dicts, key=os.path.basename): + for path in sorted(args.dicts, key=dict_zip_name): + name = dict_zip_name(path) + if args.exclude_dict_subdirs and "/" in name: + continue add_entry( archive, entries, - os.path.basename(path), + name, read_clean_dictionary(path), date_time, ) diff --git a/python-pure/.gitignore b/python-pure/.gitignore new file mode 100644 index 000000000..9efeaf1f0 --- /dev/null +++ b/python-pure/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +dist/ +build/ +opencc/opencc-resources*.zip diff --git a/python-pure/LICENSE b/python-pure/LICENSE new file mode 100644 index 000000000..1914f856a --- /dev/null +++ b/python-pure/LICENSE @@ -0,0 +1,55 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + 1. You must give any other recipients of the Work or Derivative Works a copy of this License; and + + 2. You must cause any modified files to carry prominent notices stating that You changed the files; and + + 3. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + 4. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/python-pure/MANIFEST.in b/python-pure/MANIFEST.in new file mode 100644 index 000000000..f1be660e5 --- /dev/null +++ b/python-pure/MANIFEST.in @@ -0,0 +1,3 @@ +include README.md +include README.zh-TW.md +include LICENSE diff --git a/python-pure/README.md b/python-pure/README.md new file mode 100644 index 000000000..255d3b3d4 --- /dev/null +++ b/python-pure/README.md @@ -0,0 +1,120 @@ +# opencc-py (OpenCC Pure Python Implementation) + +[繁體中文版](README.zh-TW.md) + +This directory contains a pure Python implementation of the +[OpenCC](https://github.com/BYVoid/OpenCC) Chinese conversion algorithm. It +provides the same import surface as the Python package: + +```python +import opencc + +converter = opencc.OpenCC("s2t") +print(converter.convert("汉字")) # 漢字 +``` + +## Data Dependency + +The package does not bundle OpenCC configs or dictionaries directly. Built-in +conversion data is loaded from the +[`opencc-data` PyPI package](https://pypi.org/project/opencc-data/) at runtime. + +This keeps the pure Python package small and avoids depending on generated files +under the OpenCC source tree. The converter reads: + +- config JSON files from `opencc_data.config_path()` +- dictionary text files from `opencc_data.data_path()` +- test cases from `opencc_data.test_data_path()` + +Custom config files are still supported. When a custom config references a local +dictionary path such as `CustomPhrases.ocd2`, the pure Python implementation +looks for the corresponding `CustomPhrases.txt` next to the config file. + +## Installation + +The PyPI package name is `opencc-py`. Users can install it with pip: + +```bash +python -m pip install opencc-py +``` + +For local development from this directory: + +```bash +python -m pip install . +``` + +The package version matches its `opencc-data` version and declares the matching +data package as an exact install dependency, so pip installs the compatible data +package automatically. + +Or use editable development mode: + +```bash +python -m pip install -e . +``` + +## Supported Configs + +`opencc.CONFIGS` is populated from the configs exposed by `opencc-data`. + +```python +import opencc + +print(opencc.CONFIGS) +``` + +The standard mmseg configs and configs that do not require segmentation are +supported. Jieba plugin configs are not included in `opencc-data`, so they are +not exposed as built-in configs by this package. + +## Testing + +Install test dependencies, then run pytest from the repository root: + +```bash +python -m pip install -r python-pure/tests/requirements_lock.txt +PYTHONPATH=python-pure python -m pytest python-pure/tests +``` + +The tests verify: + +- importing and initializing every built-in config +- conversion against `opencc-data` test cases +- custom config and local dictionary resolution +- golden output compatibility for supported configs + +## Differences from the Official Implementation + +This package intentionally implements only the pieces needed for pure Python +text conversion. Compared with the official C++ library and command-line tools, +it omits several lower-level details. The official Python implementation is the +[`opencc` PyPI package](https://pypi.org/project/opencc/). + +- binary dictionary loading for `.ocd2`/`.ocd`; built-in dictionaries are read + from `.txt` data supplied by `opencc-data` +- dictionary compilation and extraction tools such as `opencc_dict` and + `opencc_phrase_extract` +- the C API, shared-library loading behavior, and ABI/plugin compatibility + guarantees +- native CLI behavior, including streaming I/O, command-line option parity, and + platform-specific path handling +- package, runfiles, and source-tree data discovery fallbacks; built-in data + comes from `opencc-data` +- automatic loading of optional plugin configs or plugin resources, including + the Jieba plugin package layout +- performance optimizations from marisa-trie, Darts, and the C++ segmentation + implementation + +The conversion semantics still mirror OpenCC's config-driven pipeline: mmseg +segmentation, ordered dictionary groups, longest-prefix matching within a +dictionary, conversion chains, normalization, and optional suppression of +tofu-risk dictionaries. + +## License and Compliance + +This package is distributed under the Apache License 2.0. + +This project is a derivative work of +[OpenCC](https://github.com/BYVoid/OpenCC). Runtime conversion data is provided +by the [`opencc-data` PyPI package](https://pypi.org/project/opencc-data/). diff --git a/python-pure/README.zh-TW.md b/python-pure/README.zh-TW.md new file mode 100644 index 000000000..3614280f4 --- /dev/null +++ b/python-pure/README.zh-TW.md @@ -0,0 +1,110 @@ +# opencc-py (OpenCC 純 Python 實作) + +[English version](README.md) + +此目錄包含 [OpenCC](https://github.com/BYVoid/OpenCC) 中文轉換演算法的純 +Python 實作，提供與 Python package 相同的匯入介面： + +```python +import opencc + +converter = opencc.OpenCC("s2t") +print(converter.convert("汉字")) # 漢字 +``` + +## 資料依賴 + +此 package 不再直接內嵌 OpenCC config 或 dictionary。內建轉換資料會在執行時 +從 PyPI package [`opencc-data`](https://pypi.org/project/opencc-data/) 載入。 + +這能讓 pure Python package 保持精簡，並避免依賴 OpenCC source tree 底下的 +生成檔案。converter 會讀取： + +- `opencc_data.config_path()` 提供的 config JSON 檔案 +- `opencc_data.data_path()` 提供的 dictionary text 檔案 +- `opencc_data.test_data_path()` 提供的測試案例 + +自訂 config 仍然支援。當自訂 config 參照本地 dictionary 路徑，例如 +`CustomPhrases.ocd2`，純 Python 實作會在 config 檔案旁尋找對應的 +`CustomPhrases.txt`。 + +## 安裝 + +PyPI package 名稱是 `opencc-py`。使用者可以透過 pip 安裝： + +```bash +python -m pip install opencc-py +``` + +從此目錄進行本地開發安裝： + +```bash +python -m pip install . +``` + +此 package 的版本會與 `opencc-data` 版本一致，並將相同版本的資料 package +宣告為精確安裝依賴，因此 pip 會自動安裝相容的資料 package。 + +也可以使用 editable development mode： + +```bash +python -m pip install -e . +``` + +## 支援的 Configs + +`opencc.CONFIGS` 由 `opencc-data` 提供的 configs 產生。 + +```python +import opencc + +print(opencc.CONFIGS) +``` + +標準 mmseg configs 與不需要 segmentation 的 configs 皆受支援。Jieba plugin +configs 不包含在 `opencc-data` 中，因此此 package 不會把它們列為內建 configs。 + +## 測試 + +先安裝測試依賴，再從 repository root 執行 pytest： + +```bash +python -m pip install -r python-pure/tests/requirements_lock.txt +PYTHONPATH=python-pure python -m pytest python-pure/tests +``` + +測試會驗證： + +- 每個內建 config 都能 import 與初始化 +- 轉換結果符合 `opencc-data` 測試案例 +- 自訂 config 與本地 dictionary 解析 +- 支援 configs 的 golden output 相容性 + +## 與官方實作的差異 + +此 package 刻意只實作純 Python 文字轉換所需的部分。相較於官方 C++ library +與 command-line tools，它省略了幾個較底層的實作細節。官方 Python 實作是 PyPI +上的 [`opencc` package](https://pypi.org/project/opencc/)。 + +- `.ocd2` / `.ocd` 二進位 dictionary 載入；內建 dictionary 會讀取 + `opencc-data` 提供的 `.txt` 資料 +- `opencc_dict`、`opencc_phrase_extract` 等 dictionary 編譯與抽取工具 +- C API、shared-library 載入行為，以及 ABI/plugin 相容性保證 +- native CLI 行為，包括 streaming I/O、命令列選項完整對齊，以及平台相關路徑處理 +- package、runfiles、source-tree 資料搜尋 fallback；內建資料一律來自 + `opencc-data` +- optional plugin configs 或 plugin resources 的自動載入，包括 Jieba plugin 的 + package layout +- marisa-trie、Darts 與 C++ segmentation 實作帶來的效能最佳化 + +轉換語意仍會對齊 OpenCC 的 config-driven pipeline：mmseg segmentation、 +ordered dictionary groups、dictionary 內 longest-prefix matching、conversion +chains、normalization，以及 tofu-risk dictionaries 的可選停用。 + +## License 與合規 + +此 package 以 Apache License 2.0 發佈。 + +此專案屬於 [OpenCC](https://github.com/BYVoid/OpenCC) 的衍生作品。執行時轉換 +資料由 PyPI package [`opencc-data`](https://pypi.org/project/opencc-data/) +提供。 diff --git a/python-pure/benchmarks/benchmark_opencc.py b/python-pure/benchmarks/benchmark_opencc.py new file mode 100644 index 000000000..16067e14b --- /dev/null +++ b/python-pure/benchmarks/benchmark_opencc.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Benchmark the pure Python OpenCC package with project data.""" + +import argparse +import json +import statistics +import sys +import time +from pathlib import Path + +import opencc +from opencc import _opencc_pure + +_this_dir = Path(__file__).resolve().parent +_opencc_rootdir = _this_dir.parents[1] +_testcases_path = _opencc_rootdir / 'test' / 'testcases' / 'testcases.json' +_golden_input_path = _opencc_rootdir / 'test' / 'golden' / 'input' / 'us_constitution_zhs.txt' +_long_input_path = _opencc_rootdir / 'test' / 'benchmark' / 'zuozhuan.txt' + + +def _config_name(config): + return config[:-5] if config.endswith('.json') else config + + +def _default_configs(): + preferred = ['s2t', 't2s', 's2tw', 's2twp', 's2hk', 'tw2s'] + available = { + _config_name(config) + for config in opencc.CONFIGS + if 'jieba' not in config + } + return [config for config in preferred if config in available] + + +def _load_texts(dataset): + if dataset == 'testcases': + with open(_testcases_path, encoding='utf-8') as f: + parsed = json.load(f) + return [ + case['input'] + for case in parsed.get('cases', []) + if case.get('input') + ] + + if dataset == 'golden': + return [_golden_input_path.read_text(encoding='utf-8')] + + if dataset == 'long': + return [_long_input_path.read_text(encoding='utf-8')] + + raise ValueError(f'Unknown dataset: {dataset}') + + +def _time_call(fn): + start = time.perf_counter() + result = fn() + return time.perf_counter() - start, result + + +def _median(values): + return statistics.median(values) if values else 0.0 + + +def _benchmark_init(configs, iterations): + rows = [] + for config in configs: + samples = [] + for _ in range(iterations): + _opencc_pure._trie_cache.clear() + elapsed, _ = _time_call(lambda: opencc.OpenCC(config)) + samples.append(elapsed) + rows.append({ + 'name': f'init/{config}', + 'iterations': iterations, + 'seconds_median': _median(samples), + 'seconds_min': min(samples), + 'seconds_max': max(samples), + 'chars_per_second': None, + }) + return rows + + +def _benchmark_convert(configs, texts, warmups, iterations): + total_chars = sum(len(text) for text in texts) + rows = [] + for config in configs: + converter = opencc.OpenCC(config) + for _ in range(warmups): + for text in texts: + converter.convert(text) + + samples = [] + for _ in range(iterations): + elapsed, _ = _time_call( + lambda: [converter.convert(text) for text in texts] + ) + samples.append(elapsed) + + median = _median(samples) + rows.append({ + 'name': f'convert/{config}', + 'iterations': iterations, + 'seconds_median': median, + 'seconds_min': min(samples), + 'seconds_max': max(samples), + 'chars_per_second': total_chars / median if median > 0 else 0.0, + }) + return rows + + +def _print_table(rows): + print('| benchmark | iterations | median ms | min ms | max ms | chars/s |') + print('|---|---:|---:|---:|---:|---:|') + for row in rows: + chars_per_second = row['chars_per_second'] + if chars_per_second is None: + throughput = '' + else: + throughput = f'{chars_per_second:,.0f}' + print( + '| {name} | {iterations} | {median:.3f} | {min:.3f} | {max:.3f} | {throughput} |'.format( + name=row['name'], + iterations=row['iterations'], + median=row['seconds_median'] * 1000, + min=row['seconds_min'] * 1000, + max=row['seconds_max'] * 1000, + throughput=throughput, + ) + ) + + +def _print_json(rows): + print(json.dumps(rows, ensure_ascii=False, indent=2)) + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument( + '--dataset', + choices=['testcases', 'golden', 'long'], + default='golden', + help='Project dataset to use for conversion benchmarks.', + ) + parser.add_argument( + '--configs', + nargs='+', + default=None, + help='Config names to benchmark. Defaults to common non-jieba configs.', + ) + parser.add_argument( + '--warmups', + type=int, + default=1, + help='Warmup conversion runs before timing.', + ) + parser.add_argument( + '--iterations', + type=int, + default=5, + help='Timed iterations per benchmark.', + ) + parser.add_argument( + '--include-init', + action='store_true', + help='Also benchmark converter construction time.', + ) + parser.add_argument( + '--json', + action='store_true', + help='Emit machine-readable JSON instead of a Markdown table.', + ) + args = parser.parse_args(argv) + + configs = [_config_name(config) for config in (args.configs or _default_configs())] + texts = _load_texts(args.dataset) + + rows = [] + if args.include_init: + rows.extend(_benchmark_init(configs, args.iterations)) + rows.extend(_benchmark_convert(configs, texts, args.warmups, args.iterations)) + + if args.json: + _print_json(rows) + else: + _print_table(rows) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/python-pure/opencc/__init__.py b/python-pure/opencc/__init__.py new file mode 100644 index 000000000..cf7b437bb --- /dev/null +++ b/python-pure/opencc/__init__.py @@ -0,0 +1,13 @@ +from importlib import metadata + +from opencc._opencc_pure import OpenCC +from opencc._opencc_pure import list_configs + +__all__ = ['CONFIGS', 'OpenCC', '__version__'] + +try: + __version__ = metadata.version('opencc-py') +except metadata.PackageNotFoundError: + __version__ = 'dev' + +CONFIGS = list_configs() diff --git a/python-pure/opencc/_opencc_pure.py b/python-pure/opencc/_opencc_pure.py new file mode 100644 index 000000000..fd833704f --- /dev/null +++ b/python-pure/opencc/_opencc_pure.py @@ -0,0 +1,640 @@ +""" +Pure Python implementation of the OpenCC Chinese conversion algorithm. + +Algorithm overview (mirrors the C++ reference implementation): + + Dictionary (.txt) + Each line: ``keyvalue1 value2 ...`` + The first listed candidate is used as the conversion target. + + Trie + Character-by-character prefix tree. Built per dictionary file. + ``prefix_match(text, pos)`` returns ``(matched_length, value)`` + for the *longest* match starting at ``text[pos]``. + + DictGroup (_GroupMatcher) + An ordered list of tries. At each position the tries are tried in + declaration order; the **first** trie that has any match (not the + globally longest) wins. This gives phrase dictionaries priority + over character dictionaries, mirroring the C++ PrefixMatch logic. + + Conversion + Scans the text left-to-right. At each position the active + group/dict matcher is consulted; the matched substring is replaced + with the dictionary value. On no match, exactly one character is + passed through unchanged. + + MaxMatch Segmentation + Same scan as Conversion, but instead of replacing tokens the + matched keys are emitted as segment boundaries. Unmatched + characters accumulate in a buffer that is flushed as a single + segment when the next match is found or the string ends. + + Conversion Chain + Multiple conversion steps applied sequentially to each segment + produced by the segmenter. Each step uses an independent + group/dict matcher. +""" + +import json +import os +import importlib.util + +__all__ = ['OpenCC'] + +_opencc_data = None + + +def _get_opencc_data(): + global _opencc_data + if _opencc_data is None: + try: + import opencc_data + except ImportError as exc: + raise ImportError( + 'opencc-py pure Python resources are provided by the ' + '`opencc-data` package. Install it with `pip install opencc-data`.' + ) from exc + _opencc_data = opencc_data + return _opencc_data + + +# Trie + +class _TrieNode: + """Single node in the character trie.""" + + __slots__ = ('children', 'value') + + def __init__(self): + self.children = {} # char -> _TrieNode + self.value = None # str or None; non-None marks a terminal node + + +class _Trie: + """ + Character-by-character trie for longest-prefix matching. + + Mirrors the inner ``Table`` class in ``PrefixMatch.cpp``. + """ + + __slots__ = ('root',) + + def __init__(self): + self.root = _TrieNode() + + def add(self, key: str, value: str) -> None: + """Insert ``key -> value`` into the trie.""" + node = self.root + for ch in key: + child = node.children.get(ch) + if child is None: + child = _TrieNode() + node.children[ch] = child + node = child + node.value = value + + def prefix_match(self, text: str, pos: int): + """ + Find the longest entry whose key is a prefix of ``text[pos:]``. + + Returns ``(matched_length, value)``; ``matched_length`` is 0 when + no entry matches. + """ + node = self.root + matched_len = 0 + matched_value = None + i = pos + n = len(text) + while i < n: + child = node.children.get(text[i]) + if child is None: + break + node = child + i += 1 + if node.value is not None: + matched_len = i - pos + matched_value = node.value + return matched_len, matched_value + + def match(self, text: str, pos: int): + return self.prefix_match(text, pos) + + +# DictGroup + +class _GroupMatcher: + """ + Ordered collection of sub-matchers with a configurable match policy. + + ``short_circuit`` (default): tries sub-matchers in order; the **first** + sub-matcher that has any match returns its longest result for that entry. + Mirrors ``PrefixMatch`` with a ``short_circuit`` group policy. + + ``union``: tries all sub-matchers and returns the globally longest match + across all of them. Mirrors the ``union`` group policy. + """ + + __slots__ = ('matchers', 'policy') + + def __init__(self, matchers, policy: str = 'short_circuit'): + self.matchers = list(matchers) + self.policy = policy + + def match(self, text: str, pos: int): + """ + Return ``(matched_length, value)`` according to the group policy. + Returns ``(0, None)`` when no sub-matcher has a match. + """ + if self.policy == 'union': + best_len, best_val = 0, None + for m in self.matchers: + length, value = m.match(text, pos) + if length > best_len: + best_len, best_val = length, value + return best_len, best_val + else: # short_circuit + for m in self.matchers: + length, value = m.match(text, pos) + if length > 0: + return length, value + return 0, None + + +# Jieba segmentation + +_jieba_tokenizer_cache: dict = {} + + +class _JiebaSegmenter: + """ + Optional Jieba segmenter backed by the ``jieba`` Python package. + + The dependency is intentionally lazy: importing ``opencc`` and using the + normal mmseg configs never imports or requires ``jieba``. + """ + + __slots__ = ('tokenizer',) + + def __init__(self, resources: dict, config_dir: str = ''): + try: + import jieba + except ImportError as exc: + raise ImportError( + "Config uses Jieba segmentation. Install the optional " + "dependency with `pip install jieba` to use *_jieba configs." + ) from exc + + dict_path = _find_jieba_resource( + resources.get('dict_path', ''), + config_dir, + 'jieba.dict.utf8', + ) + user_dict_path = _find_jieba_resource( + resources.get('user_dict_path', ''), + config_dir, + 'user.dict.utf8', + required=False, + ) + + cache_key = (dict_path, user_dict_path) + tokenizer = _jieba_tokenizer_cache.get(cache_key) + if tokenizer is None: + tokenizer = jieba.Tokenizer(dictionary=dict_path) + tokenizer.initialize() + if user_dict_path: + tokenizer.load_userdict(user_dict_path) + _jieba_tokenizer_cache[cache_key] = tokenizer + self.tokenizer = tokenizer + + def segment(self, text: str) -> list: + return list(self.tokenizer.cut(text, HMM=True)) + + +def _jieba_available() -> bool: + return importlib.util.find_spec('jieba') is not None + + +def _find_jieba_resource(raw_path: str, + config_dir: str, + fallback_name: str, + required: bool = True) -> str: + """ + Resolve Jieba resources for the pure Python optional backend. + + Plugin configs point at ``jieba_merged.ocd2`` for the C++ plugin. Python + ``jieba`` consumes cppjieba's text dictionaries directly, so that request + is mapped to ``jieba.dict.utf8`` plus ``user.dict.utf8``. + """ + candidates = [] + + if raw_path: + if raw_path.endswith('jieba_merged.ocd2'): + candidates.append(raw_path[:-len('jieba_merged.ocd2')] + fallback_name) + candidates.append(fallback_name) + else: + candidates.append(raw_path) + candidates.append(os.path.join(os.path.dirname(raw_path), fallback_name)) + candidates.append(fallback_name) + else: + candidates.append(fallback_name) + + search_dirs = ( + '', + config_dir, + os.path.dirname(config_dir) if config_dir else '', + ) + + seen = set() + for candidate in candidates: + for search_dir in search_dirs: + if not candidate: + continue + path = candidate if os.path.isabs(candidate) or not search_dir else os.path.join(search_dir, candidate) + norm = os.path.abspath(path) + if norm in seen: + continue + seen.add(norm) + if os.path.isfile(path): + return path + + if required: + raise FileNotFoundError(f'Jieba resource not found: {fallback_name!r}') + return '' + + +# opencc-data resource helpers + +def _data_resource(filename: str): + return _get_opencc_data().data_path(filename or None) + + +def _config_resource(filename: str): + return _get_opencc_data().config_path(filename or None) + + +def _resource_is_file(resource) -> bool: + try: + return resource.is_file() + except (FileNotFoundError, NotADirectoryError): + return False + + +def _open_text_resource(resource): + return resource.open('r', encoding='utf-8-sig') + + +# Dictionary loading + +_trie_cache: dict = {} # cache_key -> _Trie + + +def _find_dict_txt(filename: str, config_dir: str = ''): + """ + Resolve a dictionary filename (typically ``Foo.ocd2``) to a + ``(source, needs_reverse)`` tuple. ``source`` is either a filesystem + path for custom/local dictionaries or a ``Traversable`` from + ``opencc-data`` for built-in dictionaries. + + Resolution order: + 1. Strip the extension (``Foo.ocd2`` → ``Foo``). + 2. Search custom filesystem locations when a config directory is known. + 3. Search ``opencc-data`` packaged dictionaries. + 4. If the stem ends with ``Rev`` (e.g. ``HKVariantsRev``), look for + the forward dict (``HKVariants``) and mark it for reversal. + """ + stem = filename + for ext in ('.ocd2', '.ocd', '.txt'): + if stem.endswith(ext): + stem = stem[: -len(ext)] + break + + if os.path.isabs(filename): + path = stem + '.txt' + if os.path.isfile(path): + return path, False + + search_dirs = (config_dir,) + + for search_dir in search_dirs: + if not search_dir: + continue + path = os.path.join(search_dir, stem + '.txt') + if os.path.isfile(path): + return path, False + + resource = _data_resource(stem + '.txt') + if _resource_is_file(resource): + return resource, False + + # Try reversed dict: strip the trailing "Rev" and reverse the forward file. + if stem.endswith('Rev'): + forward_stem = stem[:-3] + for search_dir in search_dirs: + if not search_dir: + continue + path = os.path.join(search_dir, forward_stem + '.txt') + if os.path.isfile(path): + return path, True + resource = _data_resource(forward_stem + '.txt') + if _resource_is_file(resource): + return resource, True + + raise FileNotFoundError(f'Dictionary file not found for: {filename!r}') + + +def _parse_dict_lines(source): + """ + Yield ``(key, candidates)`` tuples for every data line in a txt dict file. + """ + if isinstance(source, str): + f = open(source, encoding='utf-8-sig') + else: + f = _open_text_resource(source) + with f: + for line in f: + line = line.strip() + if not line or line.startswith('#') or '\t' not in line: + continue + key, rest = line.split('\t', 1) + candidates = rest.split() + if candidates: + yield key, candidates + + +def _load_trie(source, reverse: bool) -> _Trie: + """ + Load a ``keyvalue1 value2 ...`` txt file into a :class:`_Trie`. + + When ``reverse`` is ``True`` the mapping is inverted: each candidate + value becomes a key whose conversion target is the original key. When + multiple original keys share the same candidate value the first + encountered (in file order) wins, matching the behaviour of the C++ + dict-reversal script. + """ + cache_source = os.path.abspath(source) if isinstance(source, str) else str(source) + cache_key = (cache_source, reverse) + + cached = _trie_cache.get(cache_key) + if cached is not None: + return cached + + trie = _Trie() + + if not reverse: + for key, candidates in _parse_dict_lines(source): + trie.add(key, candidates[0]) + else: + # Reverse: each candidate value -> original key (first writer wins). + reversed_mapping: dict = {} + for key, candidates in _parse_dict_lines(source): + for candidate in candidates: + if candidate not in reversed_mapping: + reversed_mapping[candidate] = key + for new_key, new_value in reversed_mapping.items(): + trie.add(new_key, new_value) + + _trie_cache[cache_key] = trie + return trie + + +def _get_trie(filename: str, config_dir: str = '') -> _Trie: + """Resolve ``filename`` and return the loaded trie.""" + source, needs_reverse = _find_dict_txt(filename, config_dir) + return _load_trie(source, needs_reverse) + + +# Config parsing + +def _parse_dict_node(d: dict, config_dir: str = '', + include_tofu_risk_dicts: bool = True): + """ + Recursively parse a JSON dict-config node. + + Returns a :class:`_Trie` for leaf nodes, a :class:`_GroupMatcher` for + group nodes, or ``None`` when a dict is suppressed because + ``may_output_tofu`` is true and ``include_tofu_risk_dicts`` is false. + Mirrors ``ConfigInternal::ParseDict`` in ``Config.cpp``. + """ + dtype = d['type'] + + if dtype == 'group': + policy = d.get('match_policy', 'short_circuit') + matchers = [ + m for sub in d['dicts'] + if (m := _parse_dict_node(sub, config_dir, include_tofu_risk_dicts)) is not None + ] + if not matchers: + return None + return _GroupMatcher(matchers, policy=policy) + + if d.get('may_output_tofu', False) and not include_tofu_risk_dicts: + return None + + if dtype in ('ocd2', 'ocd', 'txt', 'text'): + return _get_trie(d['file'], config_dir) + + raise ValueError(f'Unknown dict type: {dtype!r}') + + +# Core conversion routines + +def _convert_phrase(text: str, matcher: _GroupMatcher) -> str: + """ + Convert *text* using left-to-right longest-prefix matching. + + At each position the group matcher is consulted (first-trie-wins + semantics). On a match the matched substring is replaced by the + dictionary value. On no match exactly one character is passed through + unchanged, mirroring ``Conversion::AppendConverted`` in the C++ code. + """ + parts = [] + i = 0 + n = len(text) + while i < n: + length, value = matcher.match(text, i) + if length > 0: + parts.append(value) + i += length + else: + parts.append(text[i]) + i += 1 + return ''.join(parts) + + +def _segment(text: str, matcher: _GroupMatcher) -> list: + """ + Max-match segmentation of *text*. + + Scans left-to-right using the segmentation dict. On a match the + pending character buffer is flushed as one segment and the matched key + becomes the next segment. Unmatched characters accumulate until the + next match or end of string. + + Mirrors ``MaxMatchSegmentation::Segment`` in the C++ code. + """ + segments = [] + buf = [] + i = 0 + n = len(text) + while i < n: + length, _ = matcher.match(text, i) + if length > 0: + if buf: + segments.append(''.join(buf)) + buf = [] + segments.append(text[i: i + length]) + i += length + else: + buf.append(text[i]) + i += 1 + if buf: + segments.append(''.join(buf)) + return segments + + +# Config location helpers + +def _find_config(config_name: str) -> str: + """ + Locate a config JSON file by name (with or without ``.json`` suffix). + """ + filename = config_name if config_name.endswith('.json') else config_name + '.json' + resource = _config_resource(filename) + if _resource_is_file(resource): + return resource + raise FileNotFoundError(f'Config not found: {config_name!r}') + + +def _config_seg_type(cfg: dict) -> str: + return cfg.get('segmentation', {}).get('type', '') + + +def _is_supported_cfg(cfg: dict, config_path: str = '') -> bool: + """Return True if the pure Python backend can use this parsed config.""" + seg_type = _config_seg_type(cfg) + if seg_type in ('', 'mmseg'): + return True + if seg_type == 'jieba': + if not _jieba_available(): + return False + resources = cfg.get('segmentation', {}).get('resources', {}) + config_dir = os.path.dirname(config_path) if config_path else '' + try: + _find_jieba_resource(resources.get('dict_path', ''), config_dir, 'jieba.dict.utf8') + return True + except FileNotFoundError: + return False + return False + + +def list_configs() -> list: + """ + Return a sorted list of ``.json`` config filenames available to the + pure Python converter. + """ + configs = [] + config_dir = _config_resource('') + for resource in config_dir.iterdir(): + name = resource.name + if not name.endswith('.json'): + continue + try: + cfg = json.loads(resource.read_text(encoding='utf-8')) + if _is_supported_cfg(cfg): + configs.append(name) + except Exception: + pass + return sorted(configs) + + +# Public API + +class OpenCC: + """ + Pure Python OpenCC converter. + + Usage:: + + cc = OpenCC('s2t') # or 's2t.json' + result = cc.convert('汉字') # -> '漢字' + + The ``config`` argument accepts a conversion name (e.g. ``'s2t'``) or + a bare filename (e.g. ``'s2t.json'``). Custom absolute config paths + are also accepted. + """ + + def __init__(self, config: str = 't2s', + resource_zip: 'str | None' = None, + include_tofu_risk_dictionaries: bool = True) -> None: + if resource_zip is not None: + raise ValueError( + 'resource_zip is no longer supported; install the opencc-data ' + 'package to provide built-in configs and dictionaries.' + ) + + config_name = config[:-5] if config.endswith('.json') else config + suffixed_config = config if config.endswith('.json') else f'{config}.json' + + # Load the config JSON. + # Filesystem paths take priority so that custom configs work correctly. + config_dir = '' + if os.path.isfile(config): + with open(config, encoding='utf-8') as f: + cfg = json.load(f) + config_dir = os.path.dirname(os.path.abspath(config)) + elif os.path.isfile(suffixed_config): + with open(suffixed_config, encoding='utf-8') as f: + cfg = json.load(f) + config_dir = os.path.dirname(os.path.abspath(suffixed_config)) + else: + config_resource = _find_config(config_name) + cfg = json.loads(config_resource.read_text(encoding='utf-8')) + + self.config = suffixed_config + + segmentation = cfg.get('segmentation', {}) + seg_type = segmentation.get('type', '') + include_tofu = include_tofu_risk_dictionaries + if seg_type == 'mmseg': + self._segmenter = _parse_dict_node(segmentation['dict'], config_dir, include_tofu) + elif seg_type == 'jieba': + self._segmenter = _JiebaSegmenter( + segmentation.get('resources', {}), + config_dir, + ) + elif seg_type == '': + self._segmenter = None # no segmentation; whole text is one segment + else: + raise ValueError( + f"Config {config_name!r} uses segmentation type {seg_type!r} " + f"which is not supported by the pure Python backend." + ) + + self._normalization: list = [ + m for step in cfg.get('normalization', []) + if (m := _parse_dict_node(step['dict'], config_dir, include_tofu)) is not None + ] + + self._chain: list = [ + m for step in cfg['conversion_chain'] + if (m := _parse_dict_node(step['dict'], config_dir, include_tofu)) is not None + ] + + def convert(self, text: str) -> str: + """Convert *text* and return the result.""" + for matcher in self._normalization: + text = _convert_phrase(text, matcher) + if self._segmenter is None: + segments = [text] + elif isinstance(self._segmenter, _JiebaSegmenter): + segments = self._segmenter.segment(text) + else: + segments = _segment(text, self._segmenter) + result = [] + for seg in segments: + converted = seg + for matcher in self._chain: + converted = _convert_phrase(converted, matcher) + result.append(converted) + return ''.join(result) diff --git a/python-pure/opencc/py.typed b/python-pure/opencc/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/python-pure/setup.py b/python-pure/setup.py new file mode 100644 index 000000000..a8d826cc5 --- /dev/null +++ b/python-pure/setup.py @@ -0,0 +1,72 @@ +import os +import re + +import setuptools + +_this_dir = os.path.dirname(os.path.abspath(__file__)) + +_readme_files = ( + os.path.join(_this_dir, 'README.md'), + os.path.join(_this_dir, 'README.zh-TW.md'), +) +OPENCC_DATA_VERSION = '1.3.2.dev20260628' + + +def _read_readme_for_pkg_info(path): + with open(path, encoding='utf-8') as f: + content = f.read() + content = re.sub( + r'^\[(?:English version|繁體中文版)\]\([^)]+\)\n\n', + '', + content, + flags=re.MULTILINE, + ) + return re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', content).strip() + + +def get_long_description(): + parts = [ + _read_readme_for_pkg_info(path) + for path in _readme_files + if os.path.isfile(path) + ] + return '\n\n---\n\n'.join(parts) + + +setuptools.setup( + name='opencc-py', + version=OPENCC_DATA_VERSION, + author='OpenCC contributors', + description='Conversion between Traditional and Simplified Chinese (pure Python)', + long_description=get_long_description(), + long_description_content_type='text/markdown', + url='https://github.com/BYVoid/OpenCC', + license='Apache-2.0', + license_files=['LICENSE'], + + packages=['opencc'], + package_dir={'opencc': 'opencc'}, + package_data={ + 'opencc': ['py.typed'], + }, + install_requires=[f'opencc-data=={OPENCC_DATA_VERSION}'], + + python_requires='>=3.9', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'Natural Language :: Chinese (Simplified)', + 'Natural Language :: Chinese (Traditional)', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Topic :: Scientific/Engineering', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Software Development :: Localization', + 'Topic :: Text Processing :: Linguistic', + 'Typing :: Typed', + ], + keywords=['opencc', 'convert', 'chinese'], +) diff --git a/python-pure/tests/requirements_lock.txt b/python-pure/tests/requirements_lock.txt new file mode 100644 index 000000000..b2e1f57d8 --- /dev/null +++ b/python-pure/tests/requirements_lock.txt @@ -0,0 +1,7 @@ +exceptiongroup==1.2.2 +iniconfig==2.0.0 +opencc-data==1.3.2.dev20260628 +packaging==24.1 +pluggy==1.5.0 +pygments==2.20.0 +pytest==9.0.3 diff --git a/python-pure/tests/test_opencc.py b/python-pure/tests/test_opencc.py new file mode 100644 index 000000000..c8d4f2703 --- /dev/null +++ b/python-pure/tests/test_opencc.py @@ -0,0 +1,116 @@ +import json +import os +import pytest +import sys + +import opencc_data + +_this_dir = os.path.dirname(os.path.abspath(__file__)) +_opencc_rootdir = os.path.abspath(os.path.join(_this_dir, '..', '..')) +_config_test_path = os.path.join(_opencc_rootdir, 'test', 'config_test', 'config_test.json') + + +def test_import(): + import opencc # noqa + + +def test_init_delete_converter(): + import opencc + + for config in opencc.CONFIGS: + converter = opencc.OpenCC(config) + del converter + + +def test_configs_are_loaded_from_opencc_data(): + import opencc + + expected = sorted( + resource.name + for resource in opencc_data.config_path().iterdir() + if resource.name.endswith('.json') + ) + + assert opencc.CONFIGS == expected + + +def test_jieba_configs_are_not_exposed_by_opencc_data(): + import opencc + + assert all('jieba' not in config for config in opencc.CONFIGS) + + +def test_custom_config_resolves_local_dictionaries(): + import opencc + + converter = opencc.OpenCC(_config_test_path) + + assert converter.convert('燕燕于飞，之子于归。') == '燕燕于飛，之子于歸。' + assert converter.convert('远') == '遠' + + +def test_custom_config_path_accepts_suffixless_json_path(): + import opencc + + converter = opencc.OpenCC(_config_test_path[:-5]) + + assert converter.convert('燕燕于飞，之子于归。') == '燕燕于飛，之子于歸。' + + +def test_custom_config_resolves_absolute_ocd2_path_to_text_dictionary(tmp_path): + import opencc + + dict_txt = tmp_path / 'CustomPhrases.txt' + dict_txt.write_text('测试\t測試\n', encoding='utf-8') + config_path = tmp_path / 'custom.json' + config_path.write_text(json.dumps({ + 'name': 'Custom', + 'segmentation': { + 'type': 'mmseg', + 'dict': { + 'type': 'ocd2', + 'file': str(tmp_path / 'CustomPhrases.ocd2'), + }, + }, + 'conversion_chain': [{ + 'dict': { + 'type': 'ocd2', + 'file': str(tmp_path / 'CustomPhrases.ocd2'), + }, + }], + }), encoding='utf-8') + + converter = opencc.OpenCC(str(config_path)) + + assert converter.convert('测试') == '測試' + + +def test_dictionary_parser_strips_utf8_bom(tmp_path): + from opencc import _opencc_pure + + dict_txt = tmp_path / 'BomPhrases.txt' + dict_txt.write_text('\ufeff测试\t測試\n', encoding='utf-8') + trie = _opencc_pure._load_trie(str(dict_txt), reverse=False) + + assert trie.prefix_match('测试', 0) == (2, '測試') + + +def test_conversion(): + import opencc + + with opencc_data.test_data_path('testcases.json').open('r', encoding='utf-8') as f: + parsed = json.load(f) + + for case in parsed.get('cases', []): + input_text = case.get('input') + expected = case.get('expected', {}) + if not input_text or not isinstance(expected, dict): + continue + for cfg, ans in expected.items(): + converter = opencc.OpenCC(f'{cfg}.json') + assert converter.convert(input_text) == ans, \ + 'Failed to convert {} for {} -> {}'.format(cfg, input_text, ans) + + +if __name__ == "__main__": + sys.exit(pytest.main(sys.argv[1:])) diff --git a/python-pure/tests/test_opencc_golden.py b/python-pure/tests/test_opencc_golden.py new file mode 100644 index 000000000..71c50b194 --- /dev/null +++ b/python-pure/tests/test_opencc_golden.py @@ -0,0 +1,59 @@ +import difflib +from pathlib import Path + +import pytest + +import opencc + +_this_dir = Path(__file__).resolve().parent +_opencc_rootdir = _this_dir.parents[1] +_golden_input_dir = _opencc_rootdir / 'test' / 'golden' / 'input' +_golden_output_dir = _opencc_rootdir / 'test' / 'golden' / 'output' + +_GOLDEN_INPUTS = [ + 'us_constitution_zhs.txt', +] + + +def _available_python_golden_cases(): + configs = [ + config[:-5] if config.endswith('.json') else config + for config in opencc.CONFIGS + if 'jieba' not in config + ] + cases = [] + for input_name in _GOLDEN_INPUTS: + input_stem = Path(input_name).stem + for config in configs: + expected_path = _golden_output_dir / f'{input_stem}.{config}.txt' + if expected_path.is_file(): + cases.append((input_name, config, expected_path)) + return cases + + +_GOLDEN_CASES = _available_python_golden_cases() + + +def test_python_golden_cases_are_available(): + assert _GOLDEN_CASES + + +@pytest.mark.parametrize( + 'input_name,config,expected_path', + _GOLDEN_CASES, +) +def test_python_matches_project_golden_output(input_name, config, expected_path): + input_path = _golden_input_dir / input_name + source_text = input_path.read_text(encoding='utf-8') + expected = expected_path.read_text(encoding='utf-8') + + actual = opencc.OpenCC(config).convert(source_text) + + assert actual == expected, ''.join( + difflib.unified_diff( + expected.splitlines(keepends=True), + actual.splitlines(keepends=True), + fromfile=str(expected_path), + tofile=f'opencc.OpenCC({config!r})', + ) + ) diff --git a/release-pypi-linux.sh b/release-pypi-linux.sh index 477ea787a..23feb8a05 100644 --- a/release-pypi-linux.sh +++ b/release-pypi-linux.sh @@ -1,23 +1,11 @@ #!/bin/sh set -e -# Linux wheels are built inside manylinux2014 containers managed by cibuildwheel. -python -m pip install --upgrade pip cibuildwheel twine build +python -m pip install --upgrade pip build pytest twine -export CIBW_BUILD="cp310-* cp311-* cp312-* cp313-* cp314-*" -export CIBW_SKIP="*-musllinux_*" -export CIBW_MANYLINUX_X86_64_IMAGE="manylinux2014" -export CIBW_MANYLINUX_AARCH64_IMAGE="manylinux2014" -export CIBW_ENVIRONMENT="VERSION=\"${VERSION}\"" -export CIBW_TEST_REQUIRES="pytest" -export CIBW_TEST_COMMAND="pytest {project}/python/tests" - -python -m cibuildwheel --output-dir wheelhouse - -if [ "$(uname -m)" = "x86_64" ]; then - # Build sdist only once, to avoid duplicate uploads from matrix jobs. - python -m build --sdist --outdir wheelhouse -fi +python -m build --sdist --wheel --outdir wheelhouse +python -m pip install wheelhouse/*.whl +pytest python/tests if [ "$1" != "testonly" ]; then # Upload to PyPI diff --git a/release-pypi-macos.sh b/release-pypi-macos.sh index d1b95a9dc..0edeaf079 100644 --- a/release-pypi-macos.sh +++ b/release-pypi-macos.sh @@ -1,14 +1,11 @@ #!/bin/sh set -e -python -m pip install --upgrade pip cibuildwheel twine +python -m pip install --upgrade pip build pytest twine -export CIBW_BUILD="cp310-* cp311-* cp312-* cp313-* cp314-*" -export CIBW_ENVIRONMENT="VERSION=\"${VERSION}\"" -export CIBW_TEST_REQUIRES="pytest" -export CIBW_TEST_COMMAND="pytest {project}/python/tests" - -python -m cibuildwheel --output-dir wheelhouse +python -m build --wheel --outdir wheelhouse +python -m pip install wheelhouse/*.whl +pytest python/tests if [ "$1" != "testonly" ]; then # Upload to PyPI diff --git a/release-pypi-windows.cmd b/release-pypi-windows.cmd index f2a80874b..092822059 100644 --- a/release-pypi-windows.cmd +++ b/release-pypi-windows.cmd @@ -1,16 +1,18 @@ @echo off setlocal EnableDelayedExpansion -python -m pip install --upgrade pip cibuildwheel twine +python -m pip install --upgrade pip build pytest twine if !ERRORLEVEL! NEQ 0 (EXIT !ERRORLEVEL!) -SET CIBW_BUILD=cp310-* cp311-* cp312-* cp313-* cp314-* -SET CIBW_ARCHS=AMD64 -SET CIBW_ENVIRONMENT=VERSION="%VERSION%" -SET CIBW_TEST_REQUIRES=pytest -SET CIBW_TEST_COMMAND=pytest {project}/python/tests +python -m build --wheel --outdir wheelhouse +if !ERRORLEVEL! NEQ 0 (EXIT !ERRORLEVEL!) + +for %%f in (wheelhouse\*.whl) do ( + python -m pip install "%%f" + if !ERRORLEVEL! NEQ 0 (EXIT !ERRORLEVEL!) +) -python -m cibuildwheel --output-dir wheelhouse +pytest python/tests if !ERRORLEVEL! NEQ 0 (EXIT !ERRORLEVEL!) if NOT "%~1"=="testonly" (