From 0b0a26888cab5694de3c90cde5a8fa52432e9ef5 Mon Sep 17 00:00:00 2001 From: Frank Lin Date: Fri, 3 Jul 2026 18:10:38 -0700 Subject: [PATCH] Add OPENCC_DICT_FORMAT CMake option to select dictionary output format Adds an OPENCC_DICT_FORMAT cache variable (ocd2 / text / ocd) so CMake builds can produce compiled .ocd2 dictionaries (default), plain .txt dictionaries, or legacy Darts .ocd dictionaries, with config JSON files rewritten to match the chosen format; mirrors the text-backed resource package already produced by the Bazel opencc_resources_zip target. Detailed Changes: - **CMakeLists.txt**: - Add `OPENCC_DICT_FORMAT` cache variable accepting `ocd2`, `text`, or `ocd`; selecting `ocd` without `ENABLE_DARTS=ON` fails configure with a clear error. - **data/CMakeLists.txt**: - `text`: copies raw `.txt` dictionaries into the build tree (the four bootstrap dictionaries still compile to `.ocd2` on demand for the phrase generator); rewrites and installs config JSON files via the conversion script. - `ocd`: adds a per-dictionary `.ocd` build loop running `opencc_dict --to ocd`. - Both non-default formats pass the target format to the config-conversion script. - **data/scripts/convert_config_dict_format.py** (new, replaces `convert_config_to_text.py`): - Rewrites `type`/`file` dict references in config JSON to the given target format (`text` or `ocd`); includes a matching Bazel `py_binary` target. --- CMakeLists.txt | 14 ++ data/CMakeLists.txt | 161 ++++++++++++++++++--- data/scripts/BUILD.bazel | 5 + data/scripts/convert_config_dict_format.py | 51 +++++++ 4 files changed, 213 insertions(+), 18 deletions(-) create mode 100644 data/scripts/convert_config_dict_format.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 8dd191775..0bac5282a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,20 @@ option(ENABLE_GTEST "Build all tests." OFF) option(ENABLE_BENCHMARK "Build benchmark tests." OFF) option(BUILD_OPENCC_JIEBA_PLUGIN "Build Jieba segmentation as a loadable plugin." OFF) option(BUILD_PYTHON "Build python library" OFF) + +# Built-in dictionaries and configs can be produced as compiled ocd2 +# (marisa-trie, default) or legacy ocd (Darts) dictionaries, or as plain text +# dictionaries with configs that reference them directly (slower to load, +# but human-readable and easy to interoperate with other toolchains). +set(OPENCC_DICT_FORMAT "ocd2" CACHE STRING "Format of built-in dictionaries and configs to generate/install: ocd2, ocd, or text") +set_property(CACHE OPENCC_DICT_FORMAT PROPERTY STRINGS ocd2 ocd text) +if(NOT OPENCC_DICT_FORMAT STREQUAL "ocd2" AND NOT OPENCC_DICT_FORMAT STREQUAL "ocd" AND NOT OPENCC_DICT_FORMAT STREQUAL "text") + message(FATAL_ERROR "OPENCC_DICT_FORMAT must be one of 'ocd2', 'ocd', or 'text', got '${OPENCC_DICT_FORMAT}'") +endif() +if(OPENCC_DICT_FORMAT STREQUAL "ocd" AND NOT ENABLE_DARTS) + message(FATAL_ERROR "OPENCC_DICT_FORMAT=ocd requires ENABLE_DARTS=ON (DartsDict support)") +endif() +option(USE_SYSTEM_DARTS "Use system version of Darts" OFF) option(USE_SYSTEM_GOOGLE_BENCHMARK "Use system version of Google Benchmark" OFF) option(USE_SYSTEM_GTEST "Use system version of GoogleTest" OFF) option(USE_SYSTEM_MARISA "Use system version of Marisa" OFF) diff --git a/data/CMakeLists.txt b/data/CMakeLists.txt index 1ecf03d69..5807ac104 100644 --- a/data/CMakeLists.txt +++ b/data/CMakeLists.txt @@ -9,6 +9,7 @@ set(OPENCC_BIN opencc) set(DICT_REVERSE_BIN "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/reverse.py") set(DICT_EXTRACT_TOFU_RISK_BIN "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/extract_tofu_risk.py") set(DICT_GENERATE_ST_PHRASES_BIN "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_st_phrases_from_regional_phrases.py") +set(CONFIG_CONVERT_DICT_FORMAT_BIN "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/convert_config_dict_format.py") set(DICT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dictionary) set(DICT_GENERATED_DIR ${CMAKE_CURRENT_BINARY_DIR}) @@ -44,10 +45,31 @@ set( set(DICTS ${DICTS_RAW} ${DICTS_GENERATED}) -foreach(DICT ${DICTS}) - set(DICT_TARGETS ${DICT_TARGETS} ${DICT}.ocd2) +foreach(DICT ${DICTS_RAW}) + set(DICT_${DICT}_INPUT ${DICT_DIR}/${DICT}.txt) +endforeach(DICT) + +foreach(DICT ${DICTS_GENERATED}) + set(DICT_${DICT}_INPUT ${DICT_GENERATED_DIR}/${DICT}.txt) endforeach(DICT) +# In "text" format, the Dictionaries target depends on the .txt dictionaries +# (copied into the build tree alongside the generated ones, so the build +# tree stays self-contained and runnable like the ocd2 build tree) instead of +# building/installing compiled .ocd2/.ocd dictionaries. STPhrases_GeneratedFrom +# RegionalPhrases still needs a handful of .ocd2 dictionaries to bootstrap +# itself (see below) regardless of this setting; those are built on demand +# since they remain explicit command dependencies. +if(OPENCC_DICT_FORMAT STREQUAL "text") + foreach(DICT ${DICTS}) + set(DICT_TARGETS ${DICT_TARGETS} ${DICT}.txt) + endforeach(DICT) +else() + foreach(DICT ${DICTS}) + set(DICT_TARGETS ${DICT_TARGETS} ${DICT}.${OPENCC_DICT_FORMAT}) + endforeach(DICT) +endif() + add_custom_target( Dictionaries ALL @@ -55,14 +77,6 @@ add_custom_target( ${DICT_TARGETS} ) -foreach(DICT ${DICTS_RAW}) - set(DICT_${DICT}_INPUT ${DICT_DIR}/${DICT}.txt) -endforeach(DICT) - -foreach(DICT ${DICTS_GENERATED}) - set(DICT_${DICT}_INPUT ${DICT_GENERATED_DIR}/${DICT}.txt) -endforeach(DICT) - set( DICT_TSCharactersExt_GENERATING_INPUT ${DICT_DIR}/TSCharacters.txt @@ -153,6 +167,26 @@ foreach(DICT ${DICTS_GENERATED}) ) endforeach(DICT) +if(OPENCC_DICT_FORMAT STREQUAL "text") + foreach(DICT ${DICTS_RAW}) + add_custom_command( + OUTPUT + ${DICT}.txt + COMMENT + "Copying ${DICT}.txt" + COMMAND + ${CMAKE_COMMAND} -E copy ${DICT_${DICT}_INPUT} ${DICT_GENERATED_DIR}/${DICT}.txt + DEPENDS + ${DICT_${DICT}_INPUT} + ) + set_directory_properties( + PROPERTIES + ADDITIONAL_MAKE_CLEAN_FILES + "${DICT_GENERATED_DIR}/${DICT}.txt" + ) + endforeach(DICT) +endif() + foreach(DICT ${DICTS}) add_custom_command( OUTPUT @@ -171,7 +205,7 @@ foreach(DICT ${DICTS}) ${DICT_${DICT}_INPUT} ) - if(OPENCC_ENABLE_INSTALL) + if(OPENCC_ENABLE_INSTALL AND OPENCC_DICT_FORMAT STREQUAL "ocd2") install( FILES ${DICT_GENERATED_DIR}/${DICT}.ocd2 @@ -180,6 +214,15 @@ foreach(DICT ${DICTS}) ) endif() + if(OPENCC_ENABLE_INSTALL AND OPENCC_DICT_FORMAT STREQUAL "text") + install( + FILES + ${DICT_GENERATED_DIR}/${DICT}.txt + DESTINATION + ${DIR_SHARE_OPENCC} + ) + endif() + set_directory_properties( PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES @@ -187,6 +230,42 @@ foreach(DICT ${DICTS}) ) endforeach(DICT) +if(OPENCC_DICT_FORMAT STREQUAL "ocd") + foreach(DICT ${DICTS}) + add_custom_command( + OUTPUT + ${DICT}.ocd + COMMENT + "Building ${DICT}.ocd" + COMMAND + ${OPENCC_DICT_BIN} + --input ${DICT_${DICT}_INPUT} + --output ${DICT}.ocd + --from text + --to ocd + DEPENDS + ${DICT_WIN32_DEPENDS} + ${OPENCC_DICT_BIN} + ${DICT_${DICT}_INPUT} + ) + + if(OPENCC_ENABLE_INSTALL) + install( + FILES + ${DICT_GENERATED_DIR}/${DICT}.ocd + DESTINATION + ${DIR_SHARE_OPENCC} + ) + endif() + + set_directory_properties( + PROPERTIES + ADDITIONAL_MAKE_CLEAN_FILES + "${DICT_GENERATED_DIR}/${DICT}.ocd" + ) + endforeach(DICT) +endif() + set(CONFIG_FILES config/hk2s.json config/hk2sp.json @@ -206,11 +285,57 @@ set(CONFIG_FILES config/tw2t.json ) -if(OPENCC_ENABLE_INSTALL) -install( - FILES - ${CONFIG_FILES} - DESTINATION - ${DIR_SHARE_OPENCC} -) +if(NOT OPENCC_DICT_FORMAT STREQUAL "ocd2") + set(CONFIG_CONVERTED_DIR ${CMAKE_CURRENT_BINARY_DIR}/config) + file(MAKE_DIRECTORY ${CONFIG_CONVERTED_DIR}) + + foreach(CONFIG_FILE ${CONFIG_FILES}) + get_filename_component(CONFIG_FILE_NAME ${CONFIG_FILE} NAME) + add_custom_command( + OUTPUT + ${CONFIG_CONVERTED_DIR}/${CONFIG_FILE_NAME} + COMMENT + "Converting ${CONFIG_FILE_NAME} to reference ${OPENCC_DICT_FORMAT} dictionaries" + COMMAND + ${CONFIG_CONVERT_DICT_FORMAT_BIN} + ${CMAKE_CURRENT_SOURCE_DIR}/${CONFIG_FILE} + ${CONFIG_CONVERTED_DIR}/${CONFIG_FILE_NAME} + ${OPENCC_DICT_FORMAT} + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/${CONFIG_FILE} + ${CMAKE_CURRENT_SOURCE_DIR}/scripts/convert_config_dict_format.py + ) + set(CONFIG_CONVERTED_FILES ${CONFIG_CONVERTED_FILES} ${CONFIG_CONVERTED_DIR}/${CONFIG_FILE_NAME}) + endforeach(CONFIG_FILE) + + add_custom_target( + ConvertedConfigs + ALL + DEPENDS + ${CONFIG_CONVERTED_FILES} + ) + + set_directory_properties( + PROPERTIES + ADDITIONAL_MAKE_CLEAN_FILES + "${CONFIG_CONVERTED_FILES}" + ) + + if(OPENCC_ENABLE_INSTALL) + install( + FILES + ${CONFIG_CONVERTED_FILES} + DESTINATION + ${DIR_SHARE_OPENCC} + ) + endif() +else() + if(OPENCC_ENABLE_INSTALL) + install( + FILES + ${CONFIG_FILES} + DESTINATION + ${DIR_SHARE_OPENCC} + ) + endif() endif() diff --git a/data/scripts/BUILD.bazel b/data/scripts/BUILD.bazel index 101d9acbf..2811d3c34 100644 --- a/data/scripts/BUILD.bazel +++ b/data/scripts/BUILD.bazel @@ -43,6 +43,11 @@ py_binary( deps = [":common"], ) +py_binary( + name = "convert_config_dict_format", + srcs = ["convert_config_dict_format.py"], +) + py_test( name = "common_test", size = "small", diff --git a/data/scripts/convert_config_dict_format.py b/data/scripts/convert_config_dict_format.py new file mode 100644 index 000000000..9e31921d0 --- /dev/null +++ b/data/scripts/convert_config_dict_format.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Rewrite an OpenCC config JSON so that its dictionary references point at +# a different dictionary format (text or ocd) instead of ocd2. Used to build +# the OPENCC_DICT_FORMAT=text/ocd CMake outputs, mirroring the ocd2<->text +# config conversion already used by the Bazel opencc_resources_zip / +# convert_resource_zip tools. + +import json +import sys + +TARGET_EXTENSIONS = { + "text": ".txt", + "ocd": ".ocd", +} + + +def convert_dict_references(value, target_format, target_extension): + if isinstance(value, dict): + converted = {} + for key, child in value.items(): + if key == "type" and child == "ocd2": + converted[key] = target_format + elif key == "file" and isinstance(child, str) and child.endswith(".ocd2"): + converted[key] = child[:-len(".ocd2")] + target_extension + else: + converted[key] = convert_dict_references( + child, target_format, target_extension + ) + return converted + if isinstance(value, list): + return [ + convert_dict_references(child, target_format, target_extension) + for child in value + ] + return value + + +if len(sys.argv) != 4 or sys.argv[3] not in TARGET_EXTENSIONS: + print("Rewrite ocd2 dictionary references in a config JSON to another format") + print(f"Usage: {sys.argv[0]} [input] [output] [text|ocd]") + sys.exit(1) + +target_format = sys.argv[3] +with open(sys.argv[1], encoding="utf-8") as f: + config = json.load(f) + +converted = convert_dict_references(config, target_format, TARGET_EXTENSIONS[target_format]) +with open(sys.argv[2], "w", encoding="utf-8") as f: + f.write(json.dumps(converted, ensure_ascii=False, indent=2) + "\n")