Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 125 additions & 0 deletions .github/workflows/python-wheels.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
# Build ITK Python wheels with cibuildwheel (experimental).
#
# One job per platform: ITK is compiled once inside the build container
# (CIBW_BEFORE_ALL, guarded), then each group wheel is produced by pointing
# cibuildwheel at its definition under Wrapping/Packaging/. A per-wheel matrix is
# not used because the wrapped ITK tree is too large to share across runners.
# Images, arches, and a Windows variant still need iteration in real CI.
name: Python wheels

on:
workflow_dispatch:
pull_request:
paths: ["Wrapping/Packaging/**", "Utilities/Maintenance/cibw-build-wrapped-itk.sh", ".github/workflows/python-wheels.yml"]

jobs:
wheels:
name: wheels ${{ matrix.id }}
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
- { id: linux-x86_64, runner: ubuntu-latest, archs: x86_64, itk_dir: /itk-build/build-python }
- { id: linux-aarch64, runner: ubuntu-24.04-arm, archs: aarch64, itk_dir: /itk-build/build-python }
- { id: macos-arm64, runner: macos-14, archs: arm64, itk_dir: $GITHUB_WORKSPACE/build-python }
# macos-x86_64 (macos-13) dropped: Intel macOS runners are being retired and
# the job routinely timed out in the runner queue. Re-add if x86 macOS demand returns.
# - { id: windows, runner: windows-2022, archs: AMD64, itk_dir: ... } # needs a .ps1 before-all variant

steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # ITK version is derived from git describe

- uses: actions/setup-python@v5
with: { python-version: "3.11" }

- run: python -m pip install "cibuildwheel>=2.20"

- name: Build all ITK wheels (one ITK compile, sliced per group)
shell: bash
run: |
for group in core numerics io filtering registration segmentation; do
echo "::group::itk-$group"
python -m cibuildwheel --output-dir wheelhouse "Wrapping/Packaging/$group"
echo "::endgroup::"
done
# meta ships no code but is tagged per platform+ABI (like PyPI's itk wheel),
# so it is built PER PLATFORM here too (the guarded before-all is a no-op for it).
# It has no binaries, so skip wheel repair — auditwheel/delocate error out
# with "no binary with the required architecture" on a code-less wheel.
CIBW_REPAIR_WHEEL_COMMAND="" python -m cibuildwheel --output-dir wheelhouse "Wrapping/Packaging/meta"
env:
# One abi3 wheel per platform (requires wheel.py-api="cp311"); ITK 6
# requires Python >= 3.11 (CMake/ITKSetPython3Vars.cmake).
CIBW_BUILD: "cp311-*"
# ITK ships glibc (manylinux) wheels only; musllinux is not a target and
# its Alpine/musl container cannot run the glibc vendored swig binary.
CIBW_SKIP: "*-musllinux*"
CIBW_ARCHS: ${{ matrix.archs }}
CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28

# Linux only: mount one named volume into every per-wheel container so the
# wrapped-ITK build (into /itk-build, ITK_BINARY_DIR) is compiled once and
# reused, instead of recompiled per wheel. Ignored on native macOS, which
# already reuses the build in the shared workspace.
CIBW_CONTAINER_ENGINE: "docker; create_args: --volume itk-build:/itk-build"

# Build wrapped ITK once, in-container, before the first wheel (guarded).
CIBW_BEFORE_ALL: bash Utilities/Maintenance/cibw-build-wrapped-itk.sh "{project}"

# Every wheel slices the one ITK build. {project} resolves to the
# in-container/in-checkout root; itk_dir handles the Linux-vs-native path.
CIBW_ENVIRONMENT: >-
ITK_BINARY_DIR=${{ matrix.itk_dir }}
ITK_SOURCE_DIR={project}
# ITK_BINARY_DIR is passed via CIBW_ENVIRONMENT, not here: config
# settings are not shell-expanded, so $GITHUB_WORKSPACE would stay literal.
CIBW_CONFIG_SETTINGS: >-
cmake.define.ITKPythonPackage_ITK_BINARY_REUSE=ON
cmake.define.ITKPythonPackage_USE_TBB=OFF

# cibuildwheel auto-repairs (auditwheel/delocate) — light here since ITK is static.
# No per-wheel CIBW_TEST_COMMAND: the group wheels depend on each other
# (itk-numerics needs itk-core) and are not on PyPI, so an isolated
# per-wheel install cannot resolve siblings. The whole set is smoke-tested
# together below instead.

- name: Smoke-test the full wheel set together
shell: bash
run: |
python -m pip install numpy # only external dependency of the wheel set
# Resolve the itk meta-package and its six group wheels solely from the
# local build (--no-index); numpy is already satisfied above.
python -m pip install --no-index --find-links wheelhouse itk
python -c "import itk; print(itk.Version.GetITKVersion())"

# One table per job on the run's Summary page listing every wheel produced,
# so the whole set is visible at a glance instead of only the last cibuildwheel
# invocation's output. Runs even on failure to show what was built.
- name: Wheel build summary
if: always()
shell: bash
run: |
shopt -s nullglob
wheels=(wheelhouse/*.whl)
{
echo "## ITK wheels — ${{ matrix.id }} (${#wheels[@]})"
echo ""
if [ ${#wheels[@]} -gt 0 ]; then
echo "| Wheel | Size |"
echo "|-------|------|"
for whl in "${wheels[@]}"; do
printf '| `%s` | %s |\n' "$(basename "$whl")" "$(du -h "$whl" | cut -f1)"
done
else
echo "_No wheels were produced._"
fi
} >> "$GITHUB_STEP_SUMMARY"

- uses: actions/upload-artifact@v4
with:
name: itk-wheels-${{ matrix.id }}
path: wheelhouse/*.whl
55 changes: 55 additions & 0 deletions .github/workflows/wheel-compiler-benchmark.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Benchmark: gcc vs Clang for the wrapped-ITK build (experimental).
#
# Answers "how much does Clang help the wheel build?" on Linux, the platform the
# weekly-wheel Clang switch targets (InsightSoftwareConsortium/ITK#4656, #3093).
# Two parallel jobs build the wrapped ITK tree with each compiler under
# `/usr/bin/time -v`, reporting wall-clock and peak memory. Manual trigger only.
#
# Tune before relying on it: a full wrapped build may exceed a runner's time
# limit, so this builds the default module set with wrapping (no heavy remote
# modules). Peak memory is the metric that most distinguishes the compilers on
# ITK's large generated wrapping translation units.
name: Wheel compiler benchmark

on:
workflow_dispatch:

jobs:
build:
name: wrapped-ITK (${{ matrix.compiler }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- { compiler: gcc, cc: gcc, cxx: g++ }
- { compiler: clang, cc: clang, cxx: clang++ }
steps:
- uses: actions/checkout@v4
with: { fetch-depth: 0 }

- run: sudo apt-get update && sudo apt-get install -y ${{ matrix.compiler }} ninja-build cmake

- name: Configure (${{ matrix.compiler }})
run: |
cmake -B build-bench -S . -G Ninja \
-DITK_WRAP_PYTHON:BOOL=ON \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DBUILD_TESTING:BOOL=OFF \
-DCMAKE_C_COMPILER=${{ matrix.cc }} \
-DCMAKE_CXX_COMPILER=${{ matrix.cxx }}

- name: Build (timed + peak memory)
run: /usr/bin/time -v cmake --build build-bench 2> time-${{ matrix.compiler }}.log || true

- name: Report
run: |
{
echo "### ${{ matrix.compiler }}"
grep -E 'Elapsed \(wall clock\)|Maximum resident set size' time-${{ matrix.compiler }}.log || true
} >> "$GITHUB_STEP_SUMMARY"

- uses: actions/upload-artifact@v4
with:
name: time-${{ matrix.compiler }}
path: time-${{ matrix.compiler }}.log
27 changes: 27 additions & 0 deletions Documentation/docs/contributing/python_packaging.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,33 @@ Building ITK Python wheels requires the following:
- C++ Compiler (see [scikit-build platform specific requirements](https://scikit-build.readthedocs.io/en/latest/generators.html))
- Python >= 3.10

## In-Tree Wheel Build (experimental)

ITK can also build its Python wheels directly from this repository, without the
external [ITKPythonPackage] project. The wheel definitions live in
[`Wrapping/Packaging/`](https://github.com/InsightSoftwareConsortium/ITK/tree/main/Wrapping/Packaging),
which slices a single pre-built, Python-wrapped ITK tree into the per-group
wheels (`itk-core`, `itk-numerics`, `itk-io`, `itk-filtering`,
`itk-registration`, `itk-segmentation`) plus the `itk` meta-package. ITK is
compiled once; each wheel is produced by installing only its modules' wrapping
components, with no per-wheel recompilation.

The `wheel` pixi environment provides the full flow:

```bash
# Build the wrapped ITK tree once (the reuse target every wheel slices)
pixi run -e wheel build-wrapped-itk

# Slice that tree into all ITK Python wheels
pixi run -e wheel build-wheels
```

See [`Wrapping/Packaging/README.md`](https://github.com/InsightSoftwareConsortium/ITK/blob/main/Wrapping/Packaging/README.md)
for the architecture, the per-group module placement, and platform notes. A
[cibuildwheel](https://cibuildwheel.pypa.io/) workflow that builds ITK once
inside the manylinux container and slices the wheels per platform is sketched in
`.github/workflows/python-wheels.yml`.

## Automated Platform Scripts

The following sections outline how to use the ITKPythonPackage project to build wheels on Linux, macOS, and Windows. Each script will fetch tagged ITK sources, build ITK with Python wrappings, and package binaries into the Python wheel archive format for distribution.
Expand Down
86 changes: 86 additions & 0 deletions Utilities/Maintenance/build_wheels.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""Build ITK Python wheels by slicing a pre-built, Python-wrapped ITK tree.

Run via pixi: pixi run -e wheel build-wheels
Or directly: python Utilities/Maintenance/build_wheels.py [GROUP ...]

The wheel *definitions* (per-group pyproject.toml, the slicing CMake, WHEEL_NAMES.txt)
live in Wrapping/Packaging/. This script is just the driver that loops over them.

Prerequisite: a wrapped ITK build tree configured for slicing — produce it once
with `pixi run -e wheel build-wrapped-itk` (sets WRAP_ITK_INSTALL_COMPONENT_PER_MODULE,
the PythonWheel component identifier, and a relative PY_SITE_PACKAGES_PATH).
"""
from __future__ import annotations
import argparse
import os
import subprocess
import sys
from pathlib import Path

HERE = Path(__file__).resolve().parent # Utilities/Maintenance
REPO = HERE.parents[1] # ITK source root
PACKAGING = REPO / "Wrapping" / "Packaging" # wheel definitions live here


def wheel_groups() -> list[str]:
"""Group names (e.g. 'core') from WHEEL_NAMES.txt, excluding the meta wheel."""
names = (PACKAGING / "WHEEL_NAMES.txt").read_text().split()
return [n[len("itk-"):] for n in names if n.startswith("itk-") and n != "itk-meta"]


def run_build(src: Path, dist: Path, defines: list[str]) -> None:
cmd = [
sys.executable, "-m", "build", "--wheel", "--no-isolation",
"--skip-dependency-check", "--outdir", str(dist),
*defines, str(src),
]
print(f"\n=== building {src.name} ===", flush=True)
subprocess.run(cmd, check=True)


def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("groups", nargs="*", help="group(s) to build (default: all + meta)")
ap.add_argument("--itk-build-dir",
default=os.environ.get("ITK_BINARY_DIR", str(REPO / "build-python")),
help="pre-built wrapped ITK tree to slice (default: <repo>/build-python)")
ap.add_argument("--dist", default=str(PACKAGING / "dist"), help="output directory for wheels")
ap.add_argument("--use-tbb", default="OFF", help="ITKPythonPackage_USE_TBB (ON/OFF)")
args = ap.parse_args()

itk_build = Path(args.itk_build_dir).resolve()
if not (itk_build / "cmake_install.cmake").exists():
sys.exit(
f"ERROR: '{itk_build}' is not a built ITK tree (no cmake_install.cmake).\n"
f" Build it once first: pixi run -e wheel build-wrapped-itk"
)
dist = Path(args.dist)
dist.mkdir(parents=True, exist_ok=True)

slice_defines = [
f"--config-setting=cmake.define.ITK_BINARY_DIR={itk_build}",
f"--config-setting=cmake.define.ITK_SOURCE_DIR={REPO}",
"--config-setting=cmake.define.ITKPythonPackage_ITK_BINARY_REUSE=ON",
f"--config-setting=cmake.define.ITKPythonPackage_USE_TBB={args.use_tbb}",
]

groups = args.groups or wheel_groups()
for g in groups:
run_build(
PACKAGING / g, dist,
[f"--config-setting=cmake.define.ITKPythonPackage_WHEEL_NAME=itk-{g}", *slice_defines],
)

# Build the pure-Python meta wheel only on a full run (no explicit groups given).
if not args.groups:
run_build(PACKAGING / "meta", dist, [])

print(f"\nWheels in {dist}:")
for whl in sorted(dist.glob("*.whl")):
print(f" {whl.name}")
return 0


if __name__ == "__main__":
raise SystemExit(main())
76 changes: 76 additions & 0 deletions Utilities/Maintenance/cibw-build-wrapped-itk.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
#
# cibuildwheel CIBW_BEFORE_ALL hook (SKETCH).
#
# Builds the Python-wrapped ITK tree ONCE, *inside* the build container
# (e.g. manylinux), configured for wheel slicing. Guarded so that the
# 2nd..Nth wheel builds in the same cibuildwheel job reuse it instead of
# recompiling ITK 7x. This is the "build ITK in the same image" step that
# every wheel then slices.
#
# Two toolchain paths, because the per-wheel extension (compiled later by
# scikit-build-core) must link ITK's static libs without pulling in symbols
# newer than the wheel's ABI baseline:
# * manylinux container: build ITK with the image's own gcc-toolset. The
# conda-forge compiler (via pixi) links a newer libstdc++/glibc, so
# auditwheel rejects the wheel with "too-recent versioned symbols". The
# manylinux gcc-toolset matches the compiler cibuildwheel uses for the
# extension, keeping every symbol within the manylinux_2_28 baseline.
# * macOS/native: the pixi `wheel` env supplies clang, cmake, and ninja
# (delocate handles the dylib ABI, so no glibc concern applies).
#
set -euo pipefail

PROJECT="${1:-$PWD}"
# Build tree location. On Linux, ITK_BINARY_DIR points at a Docker volume shared
# across cibuildwheel's per-wheel containers (see CIBW_CONTAINER_ENGINE), so the
# one build here is reused by every wheel instead of recompiled per container.
# On macOS the shared workspace already persists it; locally it is build-python.
BUILD="${ITK_BINARY_DIR:-${PROJECT}/build-python}"

if [ -f "${BUILD}/cmake_install.cmake" ]; then
echo "[cibw-before-all] wrapped ITK already present at ${BUILD} — reusing."
exit 0
fi

cd "${PROJECT}"

# /opt/python is the manylinux image layout; its absence means a native build.
if [ -d /opt/python ]; then
echo "[cibw-before-all] manylinux container — building ITK with the image gcc-toolset ..."

# Newest gcc-toolset on the image (manylinux_2_28 ships gcc-toolset-14). Its
# binaries stay compatible with the image's baseline glibc/libstdc++.
gts_enable=""
for f in /opt/rh/gcc-toolset-*/enable; do
[ -e "${f}" ] && gts_enable="${f}"
done
if [ -n "${gts_enable}" ]; then
# shellcheck disable=SC1090
source "${gts_enable}"
fi

# cp311 interpreter (ITK 6 needs Python >= 3.11) + pip cmake/ninja; no
# pixi/conda toolchain here.
export PATH="/opt/python/cp311-cp311/bin:${PATH}"
python -m pip install --upgrade cmake ninja

export WHEEL_CC=gcc WHEEL_CXX=g++
export WHEEL_ITK_BUILD_DIR="${BUILD}" # build into the shared volume, not /project
python "${PROJECT}/Utilities/Maintenance/configure_wrapped_itk.py"
cmake --build "${BUILD}"
else
echo "[cibw-before-all] native build via the pixi wheel env ..."
if ! command -v pixi >/dev/null 2>&1; then
echo "[cibw-before-all] pixi not found — installing it..."
export PIXI_HOME="${PIXI_HOME:-$HOME/.pixi}"
curl -fsSL https://pixi.sh/install.sh | bash
export PATH="${PIXI_HOME}/bin:${PATH}"
fi
# configure-wheels + build-wrapped-itk carry the slice flags (per-module
# components, the PythonWheel identifier, relative PY_SITE_PACKAGES_PATH) and
# pull cmake/ninja/compiler from the env.
pixi run -e wheel build-wrapped-itk
fi

echo "[cibw-before-all] wrapped ITK build complete."
Loading
Loading