diff --git a/.github/workflows/release-source.yml b/.github/workflows/release-source.yml index 2b7e08951..9af678f58 100644 --- a/.github/workflows/release-source.yml +++ b/.github/workflows/release-source.yml @@ -1,10 +1,11 @@ name: Source Package Release run-name: Source Package Release${{ !(startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '') && ' (build only)' || '' }} -# Builds the trimmed C++ source package (opencc profile: includes the Jieba -# segmentation plugin but excludes Node.js, Python, documentation, or packaging -# files) and attaches it to the release. The trimmed package keeps both build -# systems working, so it is smoke-tested with CMake and Bazel before upload. +# Builds the trimmed source packages and attaches them to the release: +# opencc C++ core and Jieba plugin only. +# bazel C++ core, Jieba plugin, and the Node.js/Python binding sources. +# Both keep the CMake and Bazel build systems working, so each is smoke-tested +# with both before upload (see scripts/build-source-package.sh). # Like the other release workflows it uploads to the draft created by # release-draft.yml; publishing is manual. @@ -28,6 +29,10 @@ jobs: runs-on: ubuntu-24.04 permissions: contents: write + strategy: + fail-fast: false + matrix: + profile: [opencc, bazel] steps: - uses: actions/checkout@v5 with: @@ -44,42 +49,22 @@ jobs: bazelisk-cache: true repository-cache: true + - name: Set up Python + if: matrix.profile == 'bazel' + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Build trimmed source package - run: | - cmake -S . -B build-src \ - -DOPENCC_SOURCE_PACKAGE_PROFILE=opencc \ - -DCPACK_SOURCE_ARCHIVE_TIMESTAMP=0 - (cd build-src && cpack --config CPackSourceConfig.cmake -G TGZ) - # Repack for reproducibility: normalize ownership (root:root), - # timestamps (epoch 0), and file order (sorted). - cpack_tarball=$(ls build-src/opencc-*-opencc-src.tar.gz) - base=$(basename "$cpack_tarball" .tar.gz) - tmpdir=$(mktemp -d) - tar xzf "$cpack_tarball" -C "$tmpdir" - mkdir -p dist - tar cf - -C "$tmpdir" --sort=name --owner=root --group=root --mtime=@0 "$base" | gzip -9n > "dist/${base}.tar.gz" - tar cf - -C "$tmpdir" --sort=name --owner=root --group=root --mtime=@0 "$base" | xz -9 > "dist/${base}.tar.xz" - rm -rf "$tmpdir" - ls -l dist/ + run: bash scripts/build-source-package.sh build ${{ matrix.profile }} - name: Smoke-test the package (CMake and Bazel) - run: | - set -e - tarball=$(ls dist/opencc-*-opencc-src.tar.gz) - workdir=$(mktemp -d) - tar xzf "$tarball" -C "$workdir" - src=$(echo "$workdir"/opencc-*-opencc-src) - # CMake: configure and build the core library, CLI tools, and Jieba plugin. - cmake -S "$src" -B "$src/build" -DCMAKE_BUILD_TYPE=Release -DBUILD_OPENCC_JIEBA_PLUGIN=ON - cmake --build "$src/build" --target opencc opencc_dict opencc-jieba --parallel - # Bazel: build the core and Jieba plugin targets. Catches a trimmed - # package that dropped a file the Bazel module or BUILD graph needs. - (cd "$src" && bazel build //src/tools:dict_converter //data/dictionary:binary_dictionaries //plugins/jieba:opencc-jieba) + run: bash scripts/build-source-package.sh smoke-test ${{ matrix.profile }} - uses: actions/upload-artifact@v4 with: - name: opencc-source-package - path: dist/opencc-*-opencc-src.tar.* + name: opencc-source-package-${{ matrix.profile }} + path: dist/opencc-*-${{ matrix.profile }}-src.tar.* - name: Upload to GitHub release if: startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '' @@ -93,6 +78,6 @@ jobs: sleep 10 done gh release upload "$tag" \ - dist/opencc-*-opencc-src.tar.gz \ - dist/opencc-*-opencc-src.tar.xz \ + dist/opencc-*-${{ matrix.profile }}-src.tar.gz \ + dist/opencc-*-${{ matrix.profile }}-src.tar.xz \ --clobber diff --git a/.gitignore b/.gitignore index fcd986ca1..445635432 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ /bazel-* /.bazelrc.user /build +/build-src /other /doc/html /dist diff --git a/AGENTS.md b/AGENTS.md index 7552717aa..a44c3b09d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,7 +54,7 @@ This document compiles the Open Chinese Convert (OpenCC) project information to ## Optional Plugin and Release Packaging - `BUILD_OPENCC_JIEBA_PLUGIN` enables the C++ jieba plugin in CMake builds. Since 1.4.1 it defaults to ON for top-level macOS builds (so Homebrew ships the plugin); other platforms, subproject builds (FetchContent / `add_subdirectory`), and Python wheel builds keep it OFF. - The merged jieba dictionary (`jieba_dict/jieba_merged.ocd2`) is generated at build time by `opencc_dict --from cppjieba_utf8`; there is no separate dictionary helper tool. Standalone plugin builds locate an installed `opencc_dict` and require OpenCC >= 1.4.1. -- Source packages: `OPENCC_SOURCE_PACKAGE_PROFILE` (`full` / `opencc`) selects which CPack source archive is produced. The `opencc` profile is the trimmed C++ package including the Jieba plugin (no Node.js, Python, docs, or packaging files) and is intentionally buildable with **both** CMake and Bazel, so Bazel workspace files (`BUILD.bazel`, `MODULE.bazel[.lock]`, `.bazelrc`, `.bazelversion`, `.bazelignore`) and the top-level `patches/` directory (referenced by `MODULE.bazel`) are kept; only `.bazelrc.user` is dropped. When editing the trimmed ignore list, do not re-exclude Bazel files or `patches/`. `release-source.yml` builds this package on a `ver.*` tag (or `workflow_dispatch` with `release_tag`), smoke-tests it with CMake and Bazel (including the Jieba plugin), repacks it for reproducibility (normalized ownership, epoch-0 timestamps, sorted entries), and uploads `.tar.gz` and `.tar.xz` to the draft release. +- Source packages: `OPENCC_SOURCE_PACKAGE_PROFILE` (`full` / `opencc` / `bazel`) selects which CPack source archive is produced. The `opencc` profile is the trimmed C++ package including the Jieba plugin (no Node.js, Python, docs, or packaging files); the `bazel` profile is the same package plus the `node/`, `python/` and `scripts/` sources, so the Bazel `//node` and `//python` targets stay buildable and testable from the tarball (npm/PyPI packaging entry points such as `package.json`, `pyproject.toml`, `setup.py` and `plugins/jieba/node/` are still dropped). Both are intentionally buildable with **both** CMake and Bazel, so Bazel workspace files (`BUILD.bazel`, `MODULE.bazel[.lock]`, `.bazelrc`, `.bazelversion`, `.bazelignore`) and the top-level `patches/` directory (referenced by `MODULE.bazel`) are kept; only `.bazelrc.user` is dropped. When editing the trimmed ignore list, do not re-exclude Bazel files or `patches/`. `scripts/build-source-package.sh` holds the build and smoke-test logic (`build` / `smoke-test` / `all` with a profile argument) and is what `release-source.yml` runs, once per profile, on a `ver.*` tag (or `workflow_dispatch` with `release_tag`): it builds the archive, repacks it for reproducibility (normalized ownership, epoch-0 timestamps, sorted entries), smoke-tests it with CMake (core, CLI tools, Jieba plugin) and Bazel (the `bazel` profile additionally runs the full CI Bazel test patterns, including `//node` and `//python`), and uploads `.tar.gz` and `.tar.xz` to the draft release. Bazel target patterns are listed explicitly instead of `//...` because the vendored `deps/googletest-1.15.0` BUILD files do not load under current Bazel versions. - `plugins/README.md` documents plugin loading, ABI expectations, and standalone plugin builds. - `scripts/release-windows-winget.ps1` is the Windows portable/WinGet release path and produces the CLI zip, checksum, and WinGet manifests. - npm release packaging is separate from the native CLI release: `opencc` and `opencc-jieba` are packed as npm `.tgz` artifacts and should be install-tested together when plugin-backed npm configs are changed. diff --git a/CMakeLists.txt b/CMakeLists.txt index d8b288077..a74e1e283 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,8 +83,8 @@ option(USE_SYSTEM_MARISA "Use system version of Marisa" OFF) option(USE_SYSTEM_PYBIND11 "Use system version of pybind11" OFF) option(USE_SYSTEM_RAPIDJSON "Use system version of RapidJSON" OFF) option(USE_SYSTEM_TCLAP "Use system version of TCLAP" OFF) -set(OPENCC_SOURCE_PACKAGE_PROFILE "full" CACHE STRING "Source package profile: full or opencc") -set_property(CACHE OPENCC_SOURCE_PACKAGE_PROFILE PROPERTY STRINGS full opencc) +set(OPENCC_SOURCE_PACKAGE_PROFILE "full" CACHE STRING "Source package profile: full, opencc or bazel") +set_property(CACHE OPENCC_SOURCE_PACKAGE_PROFILE PROPERTY STRINGS full opencc bazel) ######## Package information set (PACKAGE_URL https://github.com/BYVoid/OpenCC) @@ -105,6 +105,9 @@ set(_OPENCC_SOURCE_PACKAGE_IGNORE_FILES "") # .bazelrc, .bazelversion, .bazelignore) are intentionally NOT ignored here. # Only .bazelrc.user is excluded (it is a local/CI override that may carry a # BuildBuddy API key); it is dropped via CPACK_SOURCE_IGNORE_FILES below. +# +# Dropped by every trimmed profile: development metadata, documentation, +# third-party extras, and release packaging entry points. set(_OPENCC_TRIMMED_SOURCE_PACKAGE_IGNORE_FILES "/.appveyor.yml" "/.agents/" @@ -122,26 +125,44 @@ set(_OPENCC_TRIMMED_SOURCE_PACKAGE_IGNORE_FILES "/data/icon/" "/deps/google-benchmark/" "/deps/pybind11-2.13.1/" + # Third-party SWIG bindings (deps/marisa-*/bindings/python[3]/); the + # OpenCC python/ binding lives at the top level and is matched separately. + "/bindings/python[^/]*/" "/doc/" "/examples/" - "/node/" "/package.json" "/package-lock.json" + "/plugins/jieba/node/" "/pyproject.toml" - "/python[^/]*/" + "/python-[^/]*/" "/release-pypi-[^/]*" - "${PROJECT_SOURCE_DIR}/scripts/" "/setup.py" "/test.cmd" ) +# Node.js and Python binding sources and their build helper scripts. The +# "opencc" profile drops them; the "bazel" profile keeps them so the //node +# and //python Bazel targets stay buildable and testable from the package. +set(_OPENCC_SOURCE_PACKAGE_BINDING_IGNORE_FILES + "/node/" + "/python/" + "${PROJECT_SOURCE_DIR}/scripts/" +) if(OPENCC_SOURCE_PACKAGE_PROFILE STREQUAL "opencc") + # C++ core and Jieba plugin only. Buildable with CMake and Bazel. set(_OPENCC_SOURCE_PACKAGE_SUFFIX "-opencc-src") set(_OPENCC_SOURCE_PACKAGE_IGNORE_FILES ${_OPENCC_TRIMMED_SOURCE_PACKAGE_IGNORE_FILES} - "/plugins/jieba/node/" + ${_OPENCC_SOURCE_PACKAGE_BINDING_IGNORE_FILES} + ) +elseif(OPENCC_SOURCE_PACKAGE_PROFILE STREQUAL "bazel") + # C++ core, Jieba plugin, and the Node.js/Python binding sources: everything + # the Bazel build graph needs, without the npm/PyPI packaging entry points. + set(_OPENCC_SOURCE_PACKAGE_SUFFIX "-bazel-src") + set(_OPENCC_SOURCE_PACKAGE_IGNORE_FILES + ${_OPENCC_TRIMMED_SOURCE_PACKAGE_IGNORE_FILES} ) elseif(NOT OPENCC_SOURCE_PACKAGE_PROFILE STREQUAL "full") - message(FATAL_ERROR "OPENCC_SOURCE_PACKAGE_PROFILE must be 'full' or 'opencc'.") + message(FATAL_ERROR "OPENCC_SOURCE_PACKAGE_PROFILE must be 'full', 'opencc' or 'bazel'.") endif() if(NOT CPACK_SOURCE_PACKAGE_FILE_NAME) diff --git a/NEWS.md b/NEWS.md index c64c82eae..101aed56e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -29,7 +29,7 @@ * `packaging/debian/` 同步至 Debian 的 `1.4.1+ds1-8`。Debian 已採納上游先前提出的 jieba 插件打包提案(拆為 `opencc-jieba` 與 arch:all 的 `opencc-jieba-data` 兩個套件,內嵌 cppjieba 的理由記於 `debian/README.source`);Debian 端亦重新啟用了 autopkgtest。`packaging/debian/SPLIT_PROPOSAL.md` 隨提案落地移除。另外 `1.4.1+ds1-8` 修復了 `.ocd2` 被放進 `Architecture: all` 套件的問題:`.ocd2` 是依主機位元組序寫出的 marisa-trie 映像,所有架構都會拿到 arch:all buildd 上構建的小端序副本,opencc 在 s390x 上因而以 `size > avail_` 失敗(Debian bug [#1145010](https://bugs.debian.org/1145010),由重新啟用的 s390x autopkgtest 揪出)。`*.ocd2` 已移回 arch:any 的 `libopencc1.4`、`jieba_merged.ocd2` 移回 arch:any 的 `opencc-jieba`;`libopencc-data` 與 `opencc-jieba-data` 現在只含 JSON 配置與純文字 jieba 詞典來源。 * Debian 套件構建改在 `debian:sid` 容器中進行(同步來的打包以 unstable 為目標,Ubuntu LTS runner 無法滿足),並在 `dpkg-buildpackage` 前套用 `debian/patches` quilt series(自動跳過已在樹中的補丁);arm64 改用 `ubuntu-24.04-arm` runner 原生構建,取代先前的交叉構建([#1420](https://github.com/BYVoid/OpenCC/pull/1420))。 * 新增 `packaging/freebsd/`:FreeBSD ports tree `chinese/opencc` 條目的參考副本,供上游審閱與準備提交給 port maintainer 的改動。 - * 新增精簡 C++ 源碼包發佈流程(`release-source.yml`):以 `OPENCC_SOURCE_PACKAGE_PROFILE=opencc` 產生含 C++ 核心與 Jieba 插件的 CPack 壓縮包(不含 Node.js、Python、文檔與 packaging),並以 CMake 與 Bazel 雙路徑冒煙測試(含 Jieba 插件建置)後,重新打包以正規化檔案擁有者、時間戳與排序,產生 `.tar.gz` 與 `.tar.xz` 上傳至 draft release;為此保留 Bazel workspace 檔案與頂層 `patches/`,僅排除可能含 API key 的 `.bazelrc.user`。 + * 新增精簡源碼包發佈流程(`release-source.yml`):`OPENCC_SOURCE_PACKAGE_PROFILE` 提供 `opencc`(C++ 核心與 Jieba 插件)與 `bazel`(再加上 `node/`、`python/`、`scripts/`,可直接以 Bazel 構建並測試 `//node`、`//python`)兩種精簡設定,各自產生 CPack 壓縮包,以 CMake 與 Bazel 雙路徑冒煙測試(含 Jieba 插件建置;`bazel` 設定另跑 CI 的完整 Bazel 測試目標)後,重新打包以正規化檔案擁有者、時間戳與排序,產生 `.tar.gz` 與 `.tar.xz` 上傳至 draft release;為此保留 Bazel workspace 檔案與頂層 `patches/`,僅排除可能含 API key 的 `.bazelrc.user`。打包與冒煙測試邏輯改由 `scripts/build-source-package.sh` 承載,workflow 只負責呼叫,便於本機重現。 * 修正 `cmake/GitVersion.cmake` 的 fallback 版本號:1.4.1 發佈時仍停留在 1.4.0,導致從釋出的 tarball(無 git 資訊)構建時版本回報為 1.4.0;本次發佈同步將其更新至 1.4.2,並讓 `scripts/compute-version.sh` 的 fallback 與之一致(先前停留在 1.2.1)。因此 Debian 打包的 `0010-fix-git-version-fallback.patch` 對 1.4.2 已無作用而移除,`0012-fix-legacy-dict-detection-on-big-endian.patch` 亦已包含於本次發佈。 * Windows 發佈自本版起啟用程式碼簽章憑證:可攜式 CLI 壓縮包中的執行檔,以及 win32-x64 npm 二進位(`opencc.node`、`opencc-jieba.dll`)皆經 Authenticode 簽章,簽章服務由 [SignPath.io](https://about.signpath.io/) 免費提供、憑證由 [SignPath Foundation](https://signpath.org/) 免費提供,發佈流程改用其 release-signing policy;相關聲明與簽章/隱私政策見 README 的「Code Signing 程式碼簽章」一節([#1458](https://github.com/BYVoid/OpenCC/pull/1458), [#1459](https://github.com/BYVoid/OpenCC/pull/1459))。 * **測試**: diff --git a/scripts/build-source-package.sh b/scripts/build-source-package.sh new file mode 100755 index 000000000..91f3c7d6d --- /dev/null +++ b/scripts/build-source-package.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# +# Build and smoke-test a trimmed CPack source package. +# +# Usage: +# bash scripts/build-source-package.sh build +# bash scripts/build-source-package.sh smoke-test [tarball] +# bash scripts/build-source-package.sh all +# +# Profiles (see OPENCC_SOURCE_PACKAGE_PROFILE in CMakeLists.txt): +# opencc C++ core and Jieba plugin only. +# bazel C++ core, Jieba plugin, and the Node.js/Python binding sources. +# +# Both profiles are buildable with CMake and Bazel, so the smoke test exercises +# both build systems; the "bazel" profile additionally builds and tests the +# //node and //python targets, mirroring the target patterns used by +# .github/workflows/bazel.yml. +# +# Environment variables: +# DIST_DIR Directory to write the final archives (default: /dist) +# BUILD_DIR CMake binary directory used to run CPack (default: /build-src) +# WORK_DIR Directory the package is extracted into for the smoke test +# (default: a fresh mktemp -d) +# TAR GNU tar binary used for the reproducible repack (default: tar, +# falling back to gtar; BSD tar lacks --sort/--owner/--mtime) + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DIST_DIR="${DIST_DIR:-$ROOT_DIR/dist}" +BUILD_DIR="${BUILD_DIR:-$ROOT_DIR/build-src}" + +usage() { + sed -n '3,25p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' +} + +# Package name suffix produced by each profile (must match CMakeLists.txt). +profile_suffix() { + case "$1" in + opencc) echo "opencc-src" ;; + bazel) echo "bazel-src" ;; + *) + echo "Unknown profile: $1 (expected 'opencc' or 'bazel')" >&2 + exit 1 + ;; + esac +} + +find_tarball() { + local suffix + suffix="$(profile_suffix "$1")" + local tarball + tarball="$(ls "$DIST_DIR"/opencc-*-"$suffix".tar.gz 2>/dev/null | head -n 1 || true)" + if [ -z "$tarball" ]; then + echo "No source package found in $DIST_DIR for profile '$1'." >&2 + exit 1 + fi + echo "$tarball" +} + +# Locate a GNU tar; the reproducible repack needs --sort/--owner/--group/--mtime. +gnu_tar() { + local candidate + for candidate in "${TAR:-}" tar gtar; do + [ -n "$candidate" ] || continue + if command -v "$candidate" >/dev/null 2>&1 && "$candidate" --version 2>/dev/null | grep -q GNU; then + echo "$candidate" + return 0 + fi + done + echo "GNU tar is required for the reproducible repack (install gnu-tar and/or set TAR)." >&2 + return 1 +} + +# Build the CPack source archive, then repack it for reproducibility: +# normalized ownership (root:root), timestamps (epoch 0), and file order. +build_package() { + local profile="$1" + local suffix + suffix="$(profile_suffix "$profile")" + + cmake -S "$ROOT_DIR" -B "$BUILD_DIR" \ + -DOPENCC_SOURCE_PACKAGE_PROFILE="$profile" \ + -DCPACK_SOURCE_ARCHIVE_TIMESTAMP=0 + (cd "$BUILD_DIR" && cpack --config CPackSourceConfig.cmake -G TGZ) + + local cpack_tarball base tmpdir tar_bin + tar_bin="$(gnu_tar)" + cpack_tarball="$(ls "$BUILD_DIR"/opencc-*-"$suffix".tar.gz | head -n 1)" + base="$(basename "$cpack_tarball" .tar.gz)" + tmpdir="$(mktemp -d)" + trap 'rm -rf "$tmpdir"' RETURN + + tar xzf "$cpack_tarball" -C "$tmpdir" + mkdir -p "$DIST_DIR" + "$tar_bin" cf - -C "$tmpdir" --sort=name --owner=root --group=root --mtime=@0 "$base" | + gzip -9n >"$DIST_DIR/${base}.tar.gz" + "$tar_bin" cf - -C "$tmpdir" --sort=name --owner=root --group=root --mtime=@0 "$base" | + xz -9 >"$DIST_DIR/${base}.tar.xz" + ls -l "$DIST_DIR" +} + +smoke_test() { + local profile="$1" + local tarball="${2:-}" + [ -n "$tarball" ] || tarball="$(find_tarball "$profile")" + + local workdir src + workdir="${WORK_DIR:-$(mktemp -d)}" + mkdir -p "$workdir" + tar xzf "$tarball" -C "$workdir" + src="$(echo "$workdir"/opencc-*-"$(profile_suffix "$profile")")" + echo "Smoke-testing $tarball in $src" + + # CMake: configure and build the core library, CLI tools, and Jieba plugin. + cmake -S "$src" -B "$src/build" -DCMAKE_BUILD_TYPE=Release -DBUILD_OPENCC_JIEBA_PLUGIN=ON + cmake --build "$src/build" --target opencc opencc_dict opencc_jieba --parallel + + # Bazel: catches a trimmed package that dropped a file the Bazel module or + # BUILD graph needs. Explicit target patterns are used instead of //... + # because the vendored deps/googletest-1.15.0 BUILD files do not load under + # current Bazel versions. + if [ "$profile" = "bazel" ]; then + (cd "$src" && bazel build //:opencc //src/tools/... //plugins/... //node/...) + (cd "$src" && bazel test --skip_incompatible_explicit_targets \ + //src/... //data/... //test/... //python/... //plugins/... //node/test:node_test) + else + (cd "$src" && bazel build \ + //src/tools:dict_converter //data/dictionary:binary_dictionaries //plugins/jieba:opencc-jieba) + fi +} + +command="${1:-}" +profile="${2:-}" +case "$command" in +build) + [ -n "$profile" ] || { usage >&2; exit 1; } + build_package "$profile" + ;; +smoke-test) + [ -n "$profile" ] || { usage >&2; exit 1; } + smoke_test "$profile" "${3:-}" + ;; +all) + [ -n "$profile" ] || { usage >&2; exit 1; } + build_package "$profile" + smoke_test "$profile" + ;; +*) + usage >&2 + exit 1 + ;; +esac