Add OPENCC_DICT_FORMAT CMake option to select dictionary output format

CMakeLists.txt

@@ -51,6 +51,20 @@ option(ENABLE_GTEST "Build all tests." OFF)
 option(ENABLE_BENCHMARK "Build benchmark tests." OFF)
 option(BUILD_OPENCC_JIEBA_PLUGIN "Build Jieba segmentation as a loadable plugin." OFF)
 option(BUILD_PYTHON "Build python library" OFF)
+
+# Built-in dictionaries and configs can be produced as compiled ocd2
+# (marisa-trie, default) or legacy ocd (Darts) dictionaries, or as plain text
+# dictionaries with configs that reference them directly (slower to load,
+# but human-readable and easy to interoperate with other toolchains).
+set(OPENCC_DICT_FORMAT "ocd2" CACHE STRING "Format of built-in dictionaries and configs to generate/install: ocd2, ocd, or text")
+set_property(CACHE OPENCC_DICT_FORMAT PROPERTY STRINGS ocd2 ocd text)
+if(NOT OPENCC_DICT_FORMAT STREQUAL "ocd2" AND NOT OPENCC_DICT_FORMAT STREQUAL "ocd" AND NOT OPENCC_DICT_FORMAT STREQUAL "text")
+  message(FATAL_ERROR "OPENCC_DICT_FORMAT must be one of 'ocd2', 'ocd', or 'text', got '${OPENCC_DICT_FORMAT}'")
+endif()
+if(OPENCC_DICT_FORMAT STREQUAL "ocd" AND NOT ENABLE_DARTS)
+  message(FATAL_ERROR "OPENCC_DICT_FORMAT=ocd requires ENABLE_DARTS=ON (DartsDict support)")
+endif()
+option(USE_SYSTEM_DARTS "Use system version of Darts" OFF)
 option(USE_SYSTEM_GOOGLE_BENCHMARK "Use system version of Google Benchmark" OFF)
 option(USE_SYSTEM_GTEST "Use system version of GoogleTest" OFF)
 option(USE_SYSTEM_MARISA "Use system version of Marisa" OFF)

data/CMakeLists.txt

@@ -9,6 +9,7 @@ set(OPENCC_BIN opencc)
 set(DICT_REVERSE_BIN "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/reverse.py")
 set(DICT_EXTRACT_TOFU_RISK_BIN "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/extract_tofu_risk.py")
 set(DICT_GENERATE_ST_PHRASES_BIN "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_st_phrases_from_regional_phrases.py")
+set(CONFIG_CONVERT_DICT_FORMAT_BIN "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/convert_config_dict_format.py")
 set(DICT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dictionary)
 set(DICT_GENERATED_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
@@ -44,25 +45,38 @@ set(
 
 set(DICTS ${DICTS_RAW} ${DICTS_GENERATED})
 
-foreach(DICT ${DICTS})
-  set(DICT_TARGETS ${DICT_TARGETS} ${DICT}.ocd2)
+foreach(DICT ${DICTS_RAW})
+  set(DICT_${DICT}_INPUT ${DICT_DIR}/${DICT}.txt)
+endforeach(DICT)
+
+foreach(DICT ${DICTS_GENERATED})
+  set(DICT_${DICT}_INPUT ${DICT_GENERATED_DIR}/${DICT}.txt)
 endforeach(DICT)
 
+# In "text" format, the Dictionaries target depends on the .txt dictionaries
+# (copied into the build tree alongside the generated ones, so the build
+# tree stays self-contained and runnable like the ocd2 build tree) instead of
+# building/installing compiled .ocd2/.ocd dictionaries. STPhrases_GeneratedFrom
+# RegionalPhrases still needs a handful of .ocd2 dictionaries to bootstrap
+# itself (see below) regardless of this setting; those are built on demand
+# since they remain explicit command dependencies.
+if(OPENCC_DICT_FORMAT STREQUAL "text")
+  foreach(DICT ${DICTS})
+    set(DICT_TARGETS ${DICT_TARGETS} ${DICT}.txt)
+  endforeach(DICT)
+else()
+  foreach(DICT ${DICTS})
+    set(DICT_TARGETS ${DICT_TARGETS} ${DICT}.${OPENCC_DICT_FORMAT})
+  endforeach(DICT)
+endif()
+
 add_custom_target(
   Dictionaries
   ALL
   DEPENDS
     ${DICT_TARGETS}
 )
 
-foreach(DICT ${DICTS_RAW})
-  set(DICT_${DICT}_INPUT ${DICT_DIR}/${DICT}.txt)
-endforeach(DICT)
-
-foreach(DICT ${DICTS_GENERATED})
-  set(DICT_${DICT}_INPUT ${DICT_GENERATED_DIR}/${DICT}.txt)
-endforeach(DICT)
-
 set(
   DICT_TSCharactersExt_GENERATING_INPUT
   ${DICT_DIR}/TSCharacters.txt
@@ -153,6 +167,26 @@ foreach(DICT ${DICTS_GENERATED})
   )
 endforeach(DICT)
 
+if(OPENCC_DICT_FORMAT STREQUAL "text")
+  foreach(DICT ${DICTS_RAW})
+    add_custom_command(
+      OUTPUT
+        ${DICT}.txt
+      COMMENT
+        "Copying ${DICT}.txt"
+      COMMAND
+        ${CMAKE_COMMAND} -E copy ${DICT_${DICT}_INPUT} ${DICT_GENERATED_DIR}/${DICT}.txt
+      DEPENDS
+        ${DICT_${DICT}_INPUT}
+    )
+    set_directory_properties(
+      PROPERTIES
+        ADDITIONAL_MAKE_CLEAN_FILES
+          "${DICT_GENERATED_DIR}/${DICT}.txt"
+    )
+  endforeach(DICT)
+endif()
+
 foreach(DICT ${DICTS})
   add_custom_command(
     OUTPUT
@@ -171,7 +205,7 @@ foreach(DICT ${DICTS})
       ${DICT_${DICT}_INPUT}
   )
 
-  if(OPENCC_ENABLE_INSTALL)
+  if(OPENCC_ENABLE_INSTALL AND OPENCC_DICT_FORMAT STREQUAL "ocd2")
   install(
     FILES
       ${DICT_GENERATED_DIR}/${DICT}.ocd2
@@ -180,13 +214,58 @@ foreach(DICT ${DICTS})
   )
   endif()
 
+  if(OPENCC_ENABLE_INSTALL AND OPENCC_DICT_FORMAT STREQUAL "text")
+  install(
+    FILES
+      ${DICT_GENERATED_DIR}/${DICT}.txt
+    DESTINATION
+      ${DIR_SHARE_OPENCC}
+  )
+  endif()
+
   set_directory_properties(
     PROPERTIES
       ADDITIONAL_MAKE_CLEAN_FILES
         "${DICT_GENERATED_DIR}/${DICT}.ocd2"
   )
 endforeach(DICT)
 
+if(OPENCC_DICT_FORMAT STREQUAL "ocd")
+  foreach(DICT ${DICTS})
+    add_custom_command(
+      OUTPUT
+        ${DICT}.ocd
+      COMMENT
+        "Building ${DICT}.ocd"
+      COMMAND
+        ${OPENCC_DICT_BIN}
+          --input ${DICT_${DICT}_INPUT}
+          --output ${DICT}.ocd
+          --from text
+          --to ocd
+      DEPENDS
+        ${DICT_WIN32_DEPENDS}
+        ${OPENCC_DICT_BIN}
+        ${DICT_${DICT}_INPUT}
+    )
+
+    if(OPENCC_ENABLE_INSTALL)
+    install(
+      FILES
+        ${DICT_GENERATED_DIR}/${DICT}.ocd
+      DESTINATION
+        ${DIR_SHARE_OPENCC}
+    )
+    endif()
+
+    set_directory_properties(
+      PROPERTIES
+        ADDITIONAL_MAKE_CLEAN_FILES
+          "${DICT_GENERATED_DIR}/${DICT}.ocd"
+    )
+  endforeach(DICT)
+endif()
+
 set(CONFIG_FILES
   config/hk2s.json
   config/hk2sp.json
@@ -206,11 +285,57 @@ set(CONFIG_FILES
   config/tw2t.json
 )
 
-if(OPENCC_ENABLE_INSTALL)
-install(
-  FILES
-    ${CONFIG_FILES}
-  DESTINATION
-    ${DIR_SHARE_OPENCC}
-)
+if(NOT OPENCC_DICT_FORMAT STREQUAL "ocd2")
+  set(CONFIG_CONVERTED_DIR ${CMAKE_CURRENT_BINARY_DIR}/config)
+  file(MAKE_DIRECTORY ${CONFIG_CONVERTED_DIR})
+
+  foreach(CONFIG_FILE ${CONFIG_FILES})
+    get_filename_component(CONFIG_FILE_NAME ${CONFIG_FILE} NAME)
+    add_custom_command(
+      OUTPUT
+        ${CONFIG_CONVERTED_DIR}/${CONFIG_FILE_NAME}
+      COMMENT
+        "Converting ${CONFIG_FILE_NAME} to reference ${OPENCC_DICT_FORMAT} dictionaries"
+      COMMAND
+        ${CONFIG_CONVERT_DICT_FORMAT_BIN}
+          ${CMAKE_CURRENT_SOURCE_DIR}/${CONFIG_FILE}
+          ${CONFIG_CONVERTED_DIR}/${CONFIG_FILE_NAME}
+          ${OPENCC_DICT_FORMAT}
+      DEPENDS
+        ${CMAKE_CURRENT_SOURCE_DIR}/${CONFIG_FILE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/scripts/convert_config_dict_format.py
+    )
+    set(CONFIG_CONVERTED_FILES ${CONFIG_CONVERTED_FILES} ${CONFIG_CONVERTED_DIR}/${CONFIG_FILE_NAME})
+  endforeach(CONFIG_FILE)
+
+  add_custom_target(
+    ConvertedConfigs
+    ALL
+    DEPENDS
+      ${CONFIG_CONVERTED_FILES}
+  )
+
+  set_directory_properties(
+    PROPERTIES
+      ADDITIONAL_MAKE_CLEAN_FILES
+        "${CONFIG_CONVERTED_FILES}"
+  )
+
+  if(OPENCC_ENABLE_INSTALL)
+  install(
+    FILES
+      ${CONFIG_CONVERTED_FILES}
+    DESTINATION
+      ${DIR_SHARE_OPENCC}
+  )
+  endif()
+else()
+  if(OPENCC_ENABLE_INSTALL)
+  install(
+    FILES
+      ${CONFIG_FILES}
+    DESTINATION
+      ${DIR_SHARE_OPENCC}
+  )
+  endif()
 endif()

data/scripts/BUILD.bazel

@@ -43,6 +43,11 @@ py_binary(
     deps = [":common"],
 )
 
+py_binary(
+    name = "convert_config_dict_format",
+    srcs = ["convert_config_dict_format.py"],
+)
+
 py_test(
     name = "common_test",
     size = "small",

data/scripts/convert_config_dict_format.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Rewrite an OpenCC config JSON so that its dictionary references point at
+# a different dictionary format (text or ocd) instead of ocd2. Used to build
+# the OPENCC_DICT_FORMAT=text/ocd CMake outputs, mirroring the ocd2<->text
+# config conversion already used by the Bazel opencc_resources_zip /
+# convert_resource_zip tools.
+
+import json
+import sys
+
+TARGET_EXTENSIONS = {
+    "text": ".txt",
+    "ocd": ".ocd",
+}
+
+
+def convert_dict_references(value, target_format, target_extension):
+    if isinstance(value, dict):
+        converted = {}
+        for key, child in value.items():
+            if key == "type" and child == "ocd2":
+                converted[key] = target_format
+            elif key == "file" and isinstance(child, str) and child.endswith(".ocd2"):
+                converted[key] = child[:-len(".ocd2")] + target_extension
+            else:
+                converted[key] = convert_dict_references(
+                    child, target_format, target_extension
+                )
+        return converted
+    if isinstance(value, list):
+        return [
+            convert_dict_references(child, target_format, target_extension)
+            for child in value
+        ]
+    return value
+
+
+if len(sys.argv) != 4 or sys.argv[3] not in TARGET_EXTENSIONS:
+    print("Rewrite ocd2 dictionary references in a config JSON to another format")
+    print(f"Usage: {sys.argv[0]} [input] [output] [text|ocd]")
+    sys.exit(1)
+
+target_format = sys.argv[3]
+with open(sys.argv[1], encoding="utf-8") as f:
+    config = json.load(f)
+
+converted = convert_dict_references(config, target_format, TARGET_EXTENSIONS[target_format])
+with open(sys.argv[2], "w", encoding="utf-8") as f:
+    f.write(json.dumps(converted, ensure_ascii=False, indent=2) + "\n")