diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b89ec823..8208d73f 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,9 +2,12 @@ FROM ghcr.io/astral-sh/uv:latest AS uv FROM mcr.microsoft.com/devcontainers/base:ubuntu26.04 -# install needed packages +COPY tools/packages/apt.txt tools/packages/apt-mpi.txt /tmp/packages/ + RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install \ + && sed -e 's/#.*//' -e 's/[[:space:]]*$//' /tmp/packages/*.txt \ + | grep -v '^$' \ + | xargs apt-get -y install \ clang \ clang-tidy \ clang-format \ @@ -17,12 +20,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ lcov \ ninja-build \ python3 \ - openmpi-bin \ - libboost-dev \ - libboost-test-dev \ - libhwloc-dev \ - libmsgpack-cxx-dev \ - libopenmpi-dev \ + && rm -rf /tmp/packages \ && apt-get autoremove -y \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..54a4f2b0 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,77 @@ +name: Set up a monoprop CI environment +description: >- + Install just, uv, and the packages listed in tools/packages/. Installs tools, + not commands: anything a developer also runs belongs in the justfile. + +inputs: + python-version: + description: Python version handed to setup-uv. + required: false + default: "3.11" + cache-suffix: + description: >- + setup-uv cache key suffix. Must differ per build configuration, or a + restored cache carries a wheel built for another one. + required: true + mpi: + description: Install an MPI implementation ("on" or "off"). + required: false + default: "off" + +runs: + using: composite + steps: + - name: Install just + uses: extractions/setup-just@v4.0.0 + + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v10.0.1 + with: + enable-cache: true + cache-suffix: ${{ inputs.cache-suffix }} + python-version: ${{ inputs.python-version }} + + - name: Install system dependencies + shell: bash + env: + MPI: ${{ inputs.mpi }} + run: | + set -euo pipefail + + # macOS runners still ship bash 3.2, so no mapfile and no ${var,,}. + packages=() + add_list() { + while IFS= read -r package; do + packages+=("$package") + done < <(sed -e 's/#.*//' -e 's/[[:space:]]*$//' "$1" | grep -v '^$' || true) + } + + if [[ "$RUNNER_OS" == "macOS" ]]; then + prefix=tools/packages/brew + else + prefix=tools/packages/apt + fi + + add_list "$prefix.txt" + if [[ "$(printf '%s' "$MPI" | tr '[:upper:]' '[:lower:]')" =~ ^(on|true|yes|1)$ ]]; then + add_list "$prefix-mpi.txt" + fi + + # Derive the package from CXX, so a matrix lane needs no list of its own. + if [[ -n "${CXX:-}" ]] && ! command -v "$CXX" > /dev/null; then + case "$CXX" in + clang++*) packages+=("clang${CXX#clang++}") ;; + g++*) packages+=("g++${CXX#g++}") ;; + *) + echo "CXX=$CXX is not installed and no package name can be derived from it" >&2 + exit 2 + ;; + esac + fi + + if [[ "$RUNNER_OS" == "macOS" ]]; then + brew install "${packages[@]}" + else + sudo apt-get update + sudo apt-get install -y "${packages[@]}" + fi diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index be5c0a9d..ee7d701f 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -47,23 +47,15 @@ jobs: with: fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y just libopenmpi-dev openmpi-bin libboost-dev libhwloc-dev - - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v10.0.1 + - name: Set up the build environment + uses: ./.github/actions/setup with: - enable-cache: true cache-suffix: benchmark - python-version: "3.11" - name: Install package env: SKBUILD_CMAKE_DEFINE: monoprop_ENABLE_CXX_UNIT_TESTS=OFF - run: | - uv sync --no-progress --group bench --all-extras -v + run: just build --group bench - name: Run benchmarks env: diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index d7a2cdd0..0b9ea634 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -26,18 +26,11 @@ jobs: with: fetch-depth: 0 - - name: Install dependencies from APT - run: | - sudo apt-get update - sudo apt-get install -y just libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev libhwloc-dev - - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v10.0.1 + - name: Set up the build environment + uses: ./.github/actions/setup with: - enable-cache: true cache-suffix: copilot - python-version: "3.11" + mpi: "on" - name: Install package - run: | - uv sync --all-groups --all-extras -v + run: just build --all-groups diff --git a/.github/workflows/docpages.yml b/.github/workflows/docpages.yml index 01751a81..c7bf4897 100644 --- a/.github/workflows/docpages.yml +++ b/.github/workflows/docpages.yml @@ -43,17 +43,10 @@ jobs: # grab the history of the PR, so setuptools-scm can compute the correct version number fetch-depth: 0 - - name: Install dependencies from APT - run: | - sudo apt-get update - sudo apt-get install -y just libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev libhwloc-dev - - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v10.0.1 + - name: Set up the build environment + uses: ./.github/actions/setup with: - enable-cache: true cache-suffix: documentation - python-version: "3.11" - name: Set up Node.js uses: actions/setup-node@v7.0.0 @@ -74,11 +67,7 @@ jobs: - name: Check built docs links (including external) uses: lycheeverse/lychee-action@v2 with: - args: >- - --config .lychee.postbuild.toml --no-progress - --root-dir ${{ github.workspace }}/docs/out - --fallback-extensions html - 'docs/out/**/*.html' + args: "--config .lychee.postbuild.toml 'docs/out/**/*.html'" fail: true - name: Setup Pages diff --git a/.github/workflows/qa-analysis.yml b/.github/workflows/qa-analysis.yml index 88af1288..c022a134 100644 --- a/.github/workflows/qa-analysis.yml +++ b/.github/workflows/qa-analysis.yml @@ -54,21 +54,11 @@ jobs: with: fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y just libboost-dev libboost-test-dev libmsgpack-cxx-dev libhwloc-dev - - - name: Install MPI dependencies - if: matrix.mpi == 'on' - run: sudo apt-get install -y libopenmpi-dev openmpi-bin - - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v10.0.1 + - name: Set up the build environment + uses: ./.github/actions/setup with: - enable-cache: true cache-suffix: coverage-mpi-${{ matrix.mpi }} - python-version: "3.11" + mpi: ${{ matrix.mpi }} - name: Collect coverage run: | @@ -96,15 +86,10 @@ jobs: with: fetch-depth: 0 - - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y just - - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v10.0.1 + - name: Set up the build environment + uses: ./.github/actions/setup with: - enable-cache: true cache-suffix: coverage-report - python-version: "3.11" - name: Download coverage data uses: actions/download-artifact@v8 @@ -144,72 +129,33 @@ jobs: runs-on: ubuntu-26.04 permissions: contents: read - # Sanitizer options are per-step: only the C++ binary is fully instrumented. + # The build type also selects the tree the test recipes run against. env: SKBUILD_CMAKE_BUILD_TYPE: "AsanUbsan" SKBUILD_CMAKE_DEFINE: "monoprop_SANITIZER=asan-ubsan" - UBSAN_OPTIONS: "halt_on_error=1:print_stacktrace=1" steps: - uses: actions/checkout@v7.0.1 with: fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev libhwloc-dev - - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v10.0.1 + - name: Set up the build environment + uses: ./.github/actions/setup with: - enable-cache: true cache-suffix: gcc-asan-ubsan - python-version: "3.11" - name: Install package - run: | - uv sync --no-progress --group workspace-test --all-extras -v + run: just build --group workspace-test - # ASan needs libstdc++ preloaded too so its __cxa_throw interceptor can resolve. - - name: Resolve GCC sanitizer runtimes - run: | - echo "ASAN_PRELOAD=$(g++ -print-file-name=libasan.so):$(g++ -print-file-name=libstdc++.so.6)" >> "$GITHUB_ENV" - - # The fully instrumented binary also runs leak detection. - name: Run C++ unit tests - env: - ASAN_OPTIONS: "detect_leaks=1:leak_check_at_exit=1:detect_stack_use_after_return=1:detect_invalid_pointer_pairs=1:check_initialization_order=1:strict_init_order=1:strict_string_checks=1:halt_on_error=1" - LSAN_OPTIONS: "suppressions=${{ github.workspace }}/.github/lsan.supp" - run: | - ctest --test-dir build/editable/AsanUbsan --output-on-failure + run: just test-cpp-asan - # CPython is uninstrumented, so its leak, pointer-pair, and initialization checks are off. - # The C++ leg covers leaks. - # - # pytest replaces stderr; log files preserve sanitizer reports. - name: Run Python tests - env: - LD_PRELOAD: ${{ env.ASAN_PRELOAD }} - ASAN_OPTIONS: "detect_leaks=0:detect_stack_use_after_return=1:halt_on_error=1:log_path=${{ github.workspace }}/sanitizer-log" - UBSAN_OPTIONS: "halt_on_error=1:print_stacktrace=1:log_path=${{ github.workspace }}/sanitizer-log" - run: | - uv run pytest -r aR --durations=50 --durations-min=5.0 + run: just test-py-asan - name: Show sanitizer reports if: failure() - run: | - shopt -s nullglob - files=("${{ github.workspace }}"/sanitizer-log.*) - if [ ${#files[@]} -eq 0 ]; then - echo "No sanitizer report was written; the failure came from the tests themselves." - exit 0 - fi - for f in "${files[@]}"; do - echo "::group::$(basename "$f")" - cat "$f" - echo "::endgroup::" - done + run: just sanitizer-reports gcc-tsan: # TSan cannot load instrumented _core into stock CPython; Python coverage needs an @@ -221,33 +167,22 @@ jobs: env: SKBUILD_CMAKE_BUILD_TYPE: "Tsan" SKBUILD_CMAKE_DEFINE: "monoprop_SANITIZER=tsan" - TSAN_OPTIONS: "halt_on_error=1:history_size=4" steps: - uses: actions/checkout@v7.0.1 with: fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libopenmpi-dev openmpi-bin libboost-dev libboost-test-dev libmsgpack-cxx-dev libhwloc-dev - - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v10.0.1 + - name: Set up the build environment + uses: ./.github/actions/setup with: - enable-cache: true cache-suffix: gcc-tsan - python-version: "3.11" - name: Install package - run: | - uv sync --no-progress --group workspace-test --all-extras -v + run: just build --group workspace-test - # Restrict TSan to the concurrent partition and shared-memory paths. - name: Run C++ partition and ShmComm tests - run: | - ctest --test-dir build/editable/Tsan --output-on-failure -R "(partition_|shm_comm_)" + run: just test-cpp-tsan sonarqube-analysis: name: SonarQube analysis @@ -259,24 +194,17 @@ jobs: with: fetch-depth: 0 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libopenmpi-dev openmpi-bin libboost-dev libhwloc-dev - - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v10.0.1 + - name: Set up the build environment + uses: ./.github/actions/setup with: - enable-cache: true cache-suffix: sonarqube-mpi - python-version: "3.11" + mpi: "on" - name: Install package env: SKBUILD_CMAKE_DEFINE: "monoprop_ENABLE_CXX_UNIT_TESTS=OFF" monoprop_ENABLE_MPI: "ON" - run: | - uv sync --no-progress --all-extras -v + run: just build - name: Download coverage reports uses: actions/download-artifact@v8 @@ -309,24 +237,17 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libopenmpi-dev openmpi-bin libboost-dev libhwloc-dev - - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v10.0.1 + - name: Set up the build environment + uses: ./.github/actions/setup with: - enable-cache: true cache-suffix: clang-tidy-mpi - python-version: "3.11" + mpi: "on" - name: Build (generates compile_commands.json) env: SKBUILD_CMAKE_DEFINE: "CMAKE_CXX_COMPILER=clang++;monoprop_ENABLE_CXX_UNIT_TESTS=OFF" monoprop_ENABLE_MPI: "ON" - run: | - uv sync --no-progress --all-extras -v + run: just build - name: Register clang-tidy problem matcher run: echo "::add-matcher::.github/problem-matchers/clang-tidy.json" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2403c49f..d9933b1a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,26 +61,6 @@ jobs: with: fetch-depth: 0 - - name: Install dependencies - run: | - if [[ "${{ matrix.runner }}" == "macos-15" ]]; then - packages=(boost msgpack-cxx hwloc) - if [[ "${{ matrix.mpi }}" == "on" ]]; then - packages+=(open-mpi) - fi - brew install "${packages[@]}" - else - sudo apt-get update - packages="libboost-dev libboost-test-dev libmsgpack-cxx-dev libhwloc-dev" - if [[ "${{ matrix.mpi }}" == "on" ]]; then - packages="$packages libopenmpi-dev openmpi-bin" - fi - if [[ "${{ matrix.compiler }}" == "clang++-18" ]]; then - packages="$packages clang-18" - fi - sudo apt-get install -y $packages - fi - - name: Set build environment variables run: | if [[ "${{ matrix.runner }}" == "macos-15" ]]; then @@ -90,115 +70,25 @@ jobs: echo "CXX=${{ matrix.compiler }}" >> "$GITHUB_ENV" fi - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v10.0.1 + - name: Set up the build environment + uses: ./.github/actions/setup with: - enable-cache: true - cache-suffix: ${{ matrix.compiler }}-mpi-${{ matrix.mpi }} python-version: ${{ matrix.python-version }} + cache-suffix: ${{ matrix.compiler }}-mpi-${{ matrix.mpi }} + mpi: ${{ matrix.mpi }} - name: Install package - run: | - if [[ "${{ matrix.mpi }}" == "on" ]]; then - uv sync --no-progress --group workspace-test --all-extras -v - else - uv sync --no-progress --group workspace-test --all-extras \ - --no-extra mpi -v - fi + run: just build --group workspace-test - name: Python environment information run: | uv tree - name: Get monoprop information - run: | - uv run python <<'EOF' - import pprint - - import monoprop as mp - - pprint.pprint( - { - "version": mp.__version__, - "variant": mp.__variant__, - "compiler_flags": mp.__compiler_flags__, - "MPI": mp.has_mpi, - } - ) - EOF + run: just info - name: Verify that find_package(monoprop) works - run: | - mkdir -p find-package-smoke - - cat > find-package-smoke/CMakeLists.txt <<'EOF' - cmake_minimum_required(VERSION 3.28) - project(monoprop_find_package_smoke LANGUAGES CXX) - - set(CMAKE_CXX_EXTENSIONS OFF) - - find_package(monoprop CONFIG REQUIRED) - - add_executable(smoke main.cpp) - target_link_libraries(smoke PRIVATE monoprop::monoprop) - EOF - - cat > find-package-smoke/main.cpp <<'EOF' - #include "monoprop/MonomialPropagator.h" - #include "monoprop/detail/mpi/MPICompat.h" - - #include - #include - #include - - using namespace monoprop; - - auto main() -> int { - mpi::init(); - { - constexpr size_t kModes = 2; - OperatorDict ham; - ham[VecZ{0, 1}] = std::complex{0.0, 1.0}; - - // Graph-building / Schrodinger path: detail/graph_encoding/MPGraphEncodingStorage.h. - MonomialPropagator graph_sim(ham, - 2 * kModes, - VecZ{0, 1}, - std::optional{4U}, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt); - const std::vector monos{{0}, {1}, {2}}; - graph_sim.build_graph(monos, VecZ{0, 1, 2}, VecD{1.0, 1.0, 1.0}); - graph_sim.graph_memory_usage(); - graph_sim.expectation_value_and_gradient(VecD{0.1, 0.2, 0.3}); - - MonomialPropagator partition_sim(ham, - 2 * kModes, - VecZ{0, 1}, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - kModes, - Basis::Majorana, - 2); - partition_sim.size(); - } - mpi::finalize(); - return 0; - } - EOF - - cmake -S find-package-smoke -B find-package-smoke/build -Dmonoprop_DIR="$(uv run --no-sync python -c "import sysconfig; print(sysconfig.get_path('purelib'))")"/monoprop/cmake - - cmake --build find-package-smoke/build - - find-package-smoke/build/smoke + run: just test-find-package - name: Get test group label id: test-group-label @@ -210,57 +100,22 @@ jobs: echo "label=$runner-$pyver-$compiler-$mpi" >> $GITHUB_OUTPUT - name: Run Python tests - run: | - uv run pytest \ - -r aR \ - --durations=50 \ - --durations-min=5.0 \ - --junit-xml=pytest-"${{ steps.test-group-label.outputs.label }}".xml \ - -o junit_family=legacy + run: just test-py "${{ steps.test-group-label.outputs.label }}" - name: Run non-MPI C++ unit tests - run: | - ctest --test-dir build/editable/Release \ - --output-on-failure \ - --no-tests=error \ - --label-exclude mpi \ - --output-junit ctest-serial-"${{ steps.test-group-label.outputs.label }}".xml + run: just test-cpp "${{ steps.test-group-label.outputs.label }}" - name: Run C++ unit tests under MPI if: matrix.mpi == 'on' - # set both OpenMPI 4 (ORTE) and OpenMPI 5 (PRRTE) variants of the oversubscription setting - env: - PRTE_MCA_rmaps_default_mapping_policy: ":oversubscribe" - OMPI_MCA_rmaps_base_oversubscribe: "1" - run: | - ctest --test-dir build/editable/Release \ - --output-on-failure \ - --no-tests=error \ - --label-regex mpi \ - --output-junit ctest-mpi-"${{ steps.test-group-label.outputs.label }}".xml + run: just test-cpp-mpi "${{ steps.test-group-label.outputs.label }}" - name: Run Python tests under MPI if: matrix.mpi == 'on' - env: - OMPI_ALLOW_RUN_AS_ROOT: "1" - OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1" - OMPI_MCA_rmaps_base_oversubscribe: "1" - run: | - mpiexec --map-by :OVERSUBSCRIBE -n 2 \ - uv run --no-sync pytest tests --with-mpi -v + run: just test-py-mpi 2 - name: Run Python MPI marker rank matrix if: matrix.mpi == 'on' - env: - OMPI_ALLOW_RUN_AS_ROOT: "1" - OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1" - OMPI_MCA_rmaps_base_oversubscribe: "1" - run: | - for r in 1 2 4; do - echo "Running MPI python tests with ${r} rank(s)" - mpiexec --map-by :OVERSUBSCRIBE -n "$r" \ - uv run --no-sync pytest tests --with-mpi -m mpi -v - done + run: just test-py-mpi "1;2;4" -m mpi - name: Upload serial test results to Codecov uses: codecov/codecov-action@v7.0.0 diff --git a/.lychee.postbuild.toml b/.lychee.postbuild.toml index c9f3098f..def91982 100644 --- a/.lychee.postbuild.toml +++ b/.lychee.postbuild.toml @@ -1,5 +1,9 @@ # Post-build check over exported HTML with external links enabled. +root_dir = "docs/out" +fallback_extensions = ["html"] +no_progress = true + # Avoid noisy/generated dependency paths if ever passed as inputs. exclude_path = ["docs/node_modules/**"] diff --git a/AGENTS.md b/AGENTS.md index a1c6fe55..e8dc8567 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,9 +121,9 @@ We also use [`just`](https://github.com/casey/just) for task automation. ```bash uv sync --all-groups --all-extras -v # Build & install (workspace-wide) monoprop_ENABLE_MPI=ON uv sync --all-extras --reinstall-package monoprop --no-cache -v # MPI-enabled build -uv run pytest # Run tests (monoprop's suite + the workspace members' suites) +just test # Build, then the Python and C++ suites (test-py / test-cpp / test-cpp-mpi / test-py-mpi are the single legs) SKBUILD_CMAKE_BUILD_TYPE=AsanUbsan SKBUILD_CMAKE_DEFINE="monoprop_SANITIZER=asan-ubsan" uv sync --group workspace-test --all-extras --reinstall-package monoprop --no-cache -v # Rebuild when changing sanitizer settings. -LD_PRELOAD="$(g++ -print-file-name=libasan.so):$(g++ -print-file-name=libstdc++.so.6)" ASAN_OPTIONS=detect_leaks=0 uv run pytest # Python tests against a sanitizer tree +just test-cpp-asan / just test-py-asan / just test-cpp-tsan # Test a sanitizer tree; the recipes own the option sets and SKBUILD_CMAKE_BUILD_TYPE selects the tree just build-docs # Build documentation ``` diff --git a/README.md b/README.md index 4127244a..0d14f744 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,11 @@ uv sync --all-extras -v ctest --test-dir build/editable/Release ``` +The platform packages are listed in `tools/packages/` (`apt.txt` / `brew.txt`, +plus the `-mpi` lists), which is what CI and the devcontainer install. +`just build [uv sync args…]` performs the build CI performs — the same recipe +GitHub Actions calls, so a lane can be reproduced locally. + Full instructions — prerequisites, MPI options, and running the example executable — are in the [building guide](https://docs.monoprop.algorithmiq.tech/building). In particular, from-source builds require `hwloc` and `pkg-config` so CMake can @@ -114,7 +119,8 @@ locate `hwloc`. ```bash uv sync --all-groups --all-extras -v # installs the workspace, incl. the bench tooling -uv run python -m pytest -m "not mpi" # Python tests (serial) +just test # build, then the Python and C++ suites +just test-py / just test-cpp # one leg, against whatever is installed just test-mpi # Python + C++ tests under MPI ``` @@ -207,6 +213,9 @@ just serve-docs # live-reloading dev server just check-doc-links # checks exported HTML links (including external URLs) ``` +The link checker's options live in `.lychee.postbuild.toml`, so the recipe and the +docs workflow (which runs lychee through its own action) check the same thing. + ### Keeping documentation up to date Any PR that changes behavior, public APIs, build/test commands, or repository paths diff --git a/cpp/tests/find_package_smoke/CMakeLists.txt b/cpp/tests/find_package_smoke/CMakeLists.txt new file mode 100644 index 00000000..3cfa5e75 --- /dev/null +++ b/cpp/tests/find_package_smoke/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.28) +project(monoprop_find_package_smoke LANGUAGES CXX) + +set(CMAKE_CXX_EXTENSIONS OFF) + +find_package(monoprop CONFIG REQUIRED) + +add_executable(smoke ../link_export_probe/link_export_probe.cpp) +target_link_libraries(smoke PRIVATE monoprop::monoprop) diff --git a/docs/content/docs/building.mdx b/docs/content/docs/building.mdx index 608642bb..533f90ba 100644 --- a/docs/content/docs/building.mdx +++ b/docs/content/docs/building.mdx @@ -217,6 +217,13 @@ function reachable from the public template chain. The installed CMake target re monoprop itself was built with MPI; a serial package does not acquire MPI compile definitions or link dependencies merely because the consuming project already has an `MPI::MPI_CXX` target. +`just test-find-package` goes one step further out: it builds +`cpp/tests/find_package_smoke` — a standalone project that consumes the *installed* +package through `find_package(monoprop CONFIG)` — and runs it. That is the only leg that +exercises `monopropConfig.cmake`, so it is what catches a missing `find_dependency` or a +usage requirement the package fails to export. It compiles the same source as the in-tree +probe, so the two cannot drift. + ### Debug tree ```bash @@ -254,10 +261,7 @@ The C++ test binary is fully instrumented, so every check applies to it, including leak detection: ```bash -ASAN_OPTIONS=detect_leaks=1:detect_stack_use_after_return=1:detect_invalid_pointer_pairs=1:check_initialization_order=1:strict_init_order=1:strict_string_checks=1:halt_on_error=1 \ -LSAN_OPTIONS=suppressions=$PWD/.github/lsan.supp \ -UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \ -ctest --test-dir build/editable/AsanUbsan --output-on-failure +just test-cpp-asan ``` The Python tests are a different situation: an instrumented `_core` is loaded @@ -265,10 +269,7 @@ into an ordinary CPython, so the ASan runtime must be preloaded, and the checks that assume the whole process is instrumented have to be switched off. ```bash -ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1:halt_on_error=1 \ -UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \ -LD_PRELOAD="$(g++ -print-file-name=libasan.so):$(g++ -print-file-name=libstdc++.so.6)" \ -uv run pytest +just test-py-asan ``` Both preloads are required. Without `libasan.so`, the first `import monoprop` @@ -285,17 +286,14 @@ uninstrumented interpreter — the C++ leg above is what covers leaks. pytest replaces the stderr file descriptor, so a sanitizer report printed from native code is swallowed and the run looks like a bare exit code 1 with no -diagnostic. Add `log_path` to route reports to files instead, then read them: +diagnostic. `just test-py-asan` therefore sets `log_path`, routing the reports to +`sanitizer-log.*`; read them with: ```bash -ASAN_OPTIONS=detect_leaks=0:halt_on_error=1:log_path=$PWD/sanitizer-log \ -UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1:log_path=$PWD/sanitizer-log \ -LD_PRELOAD="$(g++ -print-file-name=libasan.so):$(g++ -print-file-name=libstdc++.so.6)" \ -uv run pytest -cat sanitizer-log.* +just sanitizer-reports ``` -The QA workflow does this and dumps the files on failure. +The QA workflow runs that recipe on failure. #### ThreadSanitizer @@ -305,8 +303,7 @@ SKBUILD_CMAKE_BUILD_TYPE=Tsan \ SKBUILD_CMAKE_DEFINE="monoprop_SANITIZER=tsan" \ uv sync --group workspace-test --all-extras --reinstall-package monoprop --no-cache -v -TSAN_OPTIONS=halt_on_error=1:history_size=4 \ -ctest --test-dir build/editable/Tsan --output-on-failure -R "(partition_|shm_comm_)" +just test-cpp-tsan ``` TSan is scoped to the concurrency surface — the partition and `ShmComm` tests — @@ -320,8 +317,17 @@ Python API under TSan would require a TSan-instrumented CPython. ### Related workflows - Use `just code-coverage` for the coverage build. -- Use `ctest --test-dir build/editable/Release -L serial` or `-L mpi-2` to - filter the discovered C++ test set. +- Use `just test-cpp` and `just test-cpp-mpi` for the two C++ legs, or + `ctest --test-dir build/editable/Release -L mpi-2` to filter further. +- Use `just build [uv sync args…]` when you want the build CI performs. It adds + `--all-extras` and, for a serial build, drops the `mpi` extra — importing + `mpi4py.MPI` without an MPI runtime raises. `monoprop_ENABLE_MPI` drives all of + it, so `monoprop_ENABLE_MPI=ON just build --group workspace-test` is the MPI + build and a bare `just build` is the serial one. +- The platform packages the build needs are listed in `tools/packages/`, one per + line: `apt.txt` / `brew.txt`, plus `apt-mpi.txt` / `brew-mpi.txt` for an MPI + build. `.github/actions/setup` and `.devcontainer/Dockerfile` both install from + those files, so there is one list per platform to keep current. ## See also diff --git a/docs/content/docs/how-to-contribute.mdx b/docs/content/docs/how-to-contribute.mdx index afc88041..2c5692f6 100644 --- a/docs/content/docs/how-to-contribute.mdx +++ b/docs/content/docs/how-to-contribute.mdx @@ -43,6 +43,12 @@ To run all pre-commit checks manually: prek run --all ``` +One of those hooks, `check-workflow-commands`, enforces where commands live: a GitHub +workflow step may not run `uv sync`, `pytest`, `ctest`, `mpiexec`, `cmake` and friends +directly, because then a failing CI lane cannot be reproduced by running what it ran. Put +the command in a `just` recipe and have the workflow call that. The few deliberate +exceptions are listed, with their reasons, in `tools/check-workflow-commands.py`. + ## Documentation Building the documentation locally requires [npm](https://docs.npmjs.com/), the Node.js package manager. Once npm is available, you can run: diff --git a/docs/content/docs/testing.mdx b/docs/content/docs/testing.mdx index 33b3cf55..18c92760 100644 --- a/docs/content/docs/testing.mdx +++ b/docs/content/docs/testing.mdx @@ -15,7 +15,8 @@ the tools needed to run the tests. The Python tests run with `pytest`. Without MPI: ```bash -uv run python -m pytest -m "not mpi" # or: just test +just test # build, then the Python and C++ suites +just test-py # the Python suite against whatever is installed ``` This collects two suites: monoprop's own (`tests/`) and the one belonging to the @@ -35,8 +36,8 @@ produce. The `just` recipes build an MPI-enabled extension and launch the suite under `mpiexec`: ```bash -just test-mpi # full suite under MPI -just test-mpi-matrix # MPI-marked tests across a rank matrix +just test-mpi # build with MPI, then run every leg +just test-py-mpi "1;2;4" -m mpi # MPI-marked tests across a rank matrix ``` To do it by hand, build with MPI on, then run under `mpiexec` with `--no-sync` @@ -105,8 +106,7 @@ CI runs the non-MPI and MPI labels separately with `--no-tests=error`, so an MPI lane fails if its MPI variant was not registered. Standalone MPI-enabled C++ test executables initialize MPI before constructing propagators and destroy the propagators before finalizing MPI. Select the groups locally with -`ctest --test-dir build/editable/Release -LE mpi` and -`ctest --test-dir build/editable/Release -L mpi --no-tests=error`. The +`just test-cpp` and `just test-cpp-mpi`, which are the same commands CI runs. The `just test-mpi '1;2;4'` recipe configures and runs the corresponding C++ CTest rank entries as well as the Python rank matrix. diff --git a/justfile b/justfile index c1a8cbcd..da08ee19 100644 --- a/justfile +++ b/justfile @@ -19,34 +19,135 @@ site := "docs" # Run the Python docs toolchain in the synced docs environment. -docs_uv := "uv run --group docs --group test --no-dev --all-extras" +docs_uv := "uv run --group docs --group test --no-dev --all-extras --no-extra mpi" +uv_sync := "uv sync --no-progress --all-extras -v" +no_mpi_extra := "--no-extra mpi" +mpi_enabled := lowercase(env('monoprop_ENABLE_MPI', 'off')) +mpi_extra := if mpi_enabled =~ '^(on|true|yes|1)$' { "" } else { no_mpi_extra } + +build_dir := "build/editable" / env('SKBUILD_CMAKE_BUILD_TYPE', 'Release') +mpiexec := "mpiexec --map-by :OVERSUBSCRIBE" + +# Keep both OpenMPI 4 (ORTE) and 5 (PRRTE) spell oversubscription env-vars. +export OMPI_MCA_rmaps_base_oversubscribe := "1" +export PRTE_MCA_rmaps_default_mapping_policy := ":oversubscribe" default: build-docs -test: - uv run python -m pytest -m "not mpi" - ctest --test-dir build/editable/Release --output-on-failure +build *ARGS: + {{ uv_sync }} {{ mpi_extra }} "$@" + +# Report what the installed extension was actually built as. + +info: + uv run --no-sync python -c 'import pprint, monoprop as mp; pprint.pprint({"version": mp.__version__, "variant": mp.__variant__, "compiler_flags": mp.__compiler_flags__, "MPI": mp.has_mpi})' + +test: build test-py test-cpp + +# The test legs below run against whatever is installed (--no-sync, so that a run cannot +# silently rebuild a differently configured wheel). LABEL, when given, names the JUnit +# report CI uploads. + +test-py LABEL='': + uv run --no-sync pytest -r aR --durations=50 --durations-min=5.0 \ + {{ if LABEL == '' { '' } else { '--junit-xml=pytest-' + LABEL + '.xml -o junit_family=legacy' } }} + +test-cpp LABEL='': + ctest --test-dir {{ build_dir }} --output-on-failure --no-tests=error --label-exclude mpi \ + {{ if LABEL == '' { '' } else { '--output-junit ctest-serial-' + LABEL + '.xml' } }} -# MPI is off by default in source builds, so build an MPI-enabled editable install -# first, then run the suite under mpiexec with --no-sync (avoids a per-rank resync). -# Pass RANKS as either a single integer or a semicolon-separated list (e.g. "1;2;4"). +# --no-tests=error: the MPI tests are registered only by an MPI-enabled build, so an +# empty set here means the build was misconfigured. -test-mpi RANKS='': - requested_ranks={{quote(RANKS)}}; \ +test-cpp-mpi LABEL='': + ctest --test-dir {{ build_dir }} --output-on-failure --no-tests=error --label-regex mpi \ + {{ if LABEL == '' { '' } else { '--output-junit ctest-mpi-' + LABEL + '.xml' } }} + +# RANKS is a single integer or a semicolon-separated list (e.g. "1;2;4"); the suite runs +# once per entry. Extra arguments go to pytest, e.g. `just test-py-mpi "1;2;4" -m mpi`. + +test-py-mpi RANKS='' *PYTEST_ARGS: + #!/usr/bin/env bash + set -euo pipefail + shift 1 + requested_ranks={{ quote(RANKS) }} + ranks="${requested_ranks:-${monoprop_MPI_TEST_PROCS:-2}}" + for r in ${ranks//;/ }; do + echo "Running the Python test suite with ${r} MPI rank(s)" + {{ mpiexec }} -n "$r" uv run --no-sync pytest tests --with-mpi -v "$@" + done + +# Build MPI-enabled, then run every leg. The C++ +# rank matrix is a build-time setting. + +test-mpi RANKS='': && (test-py-mpi RANKS) test-cpp test-cpp-mpi + requested_ranks={{ quote(RANKS) }}; \ ranks="${requested_ranks:-${monoprop_MPI_TEST_PROCS:-2}}"; \ - monoprop_ENABLE_MPI=ON uv sync --all-extras --group workspace-test \ - --reinstall-package monoprop --no-cache -v \ - --config-settings-package="monoprop:cmake.define.monoprop_MPI_TEST_PROCS=${ranks}"; \ - export OMPI_MCA_rmaps_base_oversubscribe="1"; \ - export PRTE_MCA_rmaps_default_mapping_policy=":oversubscribe"; \ - for r in ${ranks//;/ }; \ - do echo "Running full Python test suite with ${r} MPI rank(s)"; \ - mpiexec -n "$r" uv run --no-sync python -m pytest tests --with-mpi -v; \ - done; \ - echo "Running non-MPI C++ unit tests"; \ - ctest --test-dir build/editable/Release --output-on-failure --no-tests=error --label-exclude mpi; \ - echo "Running configured C++ MPI rank matrix: ${ranks}"; \ - ctest --test-dir build/editable/Release --output-on-failure --no-tests=error --label-regex mpi + monoprop_ENABLE_MPI=ON {{ uv_sync }} --group workspace-test \ + --reinstall-package monoprop --no-cache \ + --config-settings-package="monoprop:cmake.define.monoprop_MPI_TEST_PROCS=${ranks}" + +# Build and run a consumer project against the installed package. + +test-find-package BUILD_DIR='build/find-package-smoke': + #!/usr/bin/env bash + set -euo pipefail + build_dir={{ quote(BUILD_DIR) }} + site_packages="$(uv run --no-sync python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')" + cmake -S cpp/tests/find_package_smoke -B "$build_dir" \ + -Dmonoprop_DIR="$site_packages/monoprop/cmake" + cmake --build "$build_dir" + "$build_dir/smoke" + +# The sanitizer legs run against a tree built with SKBUILD_CMAKE_BUILD_TYPE=AsanUbsan (or +# Tsan) and the matching monoprop_SANITIZER define. +# +# Only the C++ binary is fully instrumented, so the option sets differ per leg and cannot +# be hoisted to the environment. + +ubsan_options := "halt_on_error=1:print_stacktrace=1" +sanitizer_log := project_source_dir / "sanitizer-log" + + +test-cpp-asan: + ASAN_OPTIONS="detect_leaks=1:leak_check_at_exit=1:detect_stack_use_after_return=1:detect_invalid_pointer_pairs=1:check_initialization_order=1:strict_init_order=1:strict_string_checks=1:halt_on_error=1" \ + LSAN_OPTIONS="suppressions={{ project_source_dir }}/.github/lsan.supp" \ + UBSAN_OPTIONS="{{ ubsan_options }}" \ + ctest --test-dir {{ build_dir }} --output-on-failure + +# CPython is uninstrumented, so the leak, pointer-pair and initialization checks are off +# here; the C++ leg covers those. ASan needs libstdc++ preloaded too, or its __cxa_throw +# interceptor does not resolve. pytest replaces stderr, so the reports go to log files. + +test-py-asan: + LD_PRELOAD="$(g++ -print-file-name=libasan.so):$(g++ -print-file-name=libstdc++.so.6)" \ + ASAN_OPTIONS="detect_leaks=0:detect_stack_use_after_return=1:halt_on_error=1:log_path={{ sanitizer_log }}" \ + UBSAN_OPTIONS="{{ ubsan_options }}:log_path={{ sanitizer_log }}" \ + uv run --no-sync pytest -r aR --durations=50 --durations-min=5.0 + +# Print what test-py-asan sent to the log files. Silent when the tests themselves failed. + +sanitizer-reports: + #!/usr/bin/env bash + set -euo pipefail + shopt -s nullglob + files=({{ sanitizer_log }}.*) + if (( ${#files[@]} == 0 )); then + echo "No sanitizer report was written; the failure came from the tests themselves." + exit 0 + fi + for f in "${files[@]}"; do + echo "::group::$(basename "$f")" + cat "$f" + echo "::endgroup::" + done + +# TSan cannot load an instrumented _core into stock CPython, so this leg is C++ only, and +# restricted to the concurrent partition and shared-memory paths. + +test-cpp-tsan: + TSAN_OPTIONS="halt_on_error=1:history_size=4" \ + ctest --test-dir {{ build_dir }} --output-on-failure -R "(partition_|shm_comm_)" # Collect one instrumented build. MPI must be "on" or "off"; each variant needs its own build # and output directories because the preprocessor selects different compatibility paths. @@ -55,25 +156,20 @@ code-coverage-collect MPI BUILD_DIR OUTPUT_DIR: #!/usr/bin/env bash set -euo pipefail - mpi={{quote(MPI)}} - build_dir={{quote(BUILD_DIR)}} - output_dir={{quote(OUTPUT_DIR)}} + mpi={{ quote(MPI) }} + build_dir={{ quote(BUILD_DIR) }} + output_dir={{ quote(OUTPUT_DIR) }} export GCOV_EXIT_AT_ERROR=1 if [[ "$mpi" != "on" && "$mpi" != "off" ]]; then echo "MPI must be 'on' or 'off', got: $mpi" >&2 exit 2 fi - sync_args=( - --no-progress - --group workspace-test - --all-extras - --reinstall-package monoprop - -v - ) + sync_args=({{ uv_sync }}) if [[ "$mpi" == "off" ]]; then - sync_args+=(--no-extra mpi) + sync_args+=({{ no_mpi_extra }}) fi + sync_args+=(--group workspace-test --reinstall-package monoprop) # The top-level recipe isolates local rebuilds from a wheel cached for the other variant. if [[ "${monoprop_COVERAGE_NO_CACHE:-OFF}" == "ON" ]]; then sync_args+=(--no-cache) @@ -82,7 +178,7 @@ code-coverage-collect MPI BUILD_DIR OUTPUT_DIR: SKBUILD_BUILD_DIR="$build_dir" \ SKBUILD_CMAKE_BUILD_TYPE=Coverage \ monoprop_ENABLE_MPI="$mpi" \ - uv sync "${sync_args[@]}" + "${sync_args[@]}" rm -rf "$output_dir" mkdir -p "$output_dir" @@ -91,11 +187,10 @@ code-coverage-collect MPI BUILD_DIR OUTPUT_DIR: uv run --no-sync coverage run --parallel-mode -m pytest -m "not mpi" if [[ "$mpi" == "on" ]]; then + # The coverage lane runs in a container as root. export OMPI_ALLOW_RUN_AS_ROOT=1 export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - export OMPI_MCA_rmaps_base_oversubscribe=1 - export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe - mpiexec --map-by :OVERSUBSCRIBE -n 2 \ + {{ mpiexec }} -n 2 \ uv run --no-sync coverage run --parallel-mode \ -m pytest tests --with-mpi -m mpi fi @@ -167,9 +262,9 @@ code-coverage-aggregate SERIAL_DIR MPI_DIR OUTPUT_DIR='.': #!/usr/bin/env bash set -euo pipefail - serial_dir={{quote(SERIAL_DIR)}} - mpi_dir={{quote(MPI_DIR)}} - output_dir={{quote(OUTPUT_DIR)}} + serial_dir={{ quote(SERIAL_DIR) }} + mpi_dir={{ quote(MPI_DIR) }} + output_dir={{ quote(OUTPUT_DIR) }} mkdir -p "$output_dir" rm -f "$output_dir/.coverage" \ "$output_dir/python-coverage.xml" \ @@ -213,11 +308,11 @@ code-coverage-aggregate SERIAL_DIR MPI_DIR OUTPUT_DIR='.': code-coverage-html REPORT_DIR='.' OUTPUT_DIR='monoprop-coverage': lcov \ --ignore-errors inconsistent,corrupt \ - -a {{quote(REPORT_DIR)}}/python-coverage.info \ - -a {{quote(REPORT_DIR)}}/cpp-coverage-through-python-bindings.info \ - -a {{quote(REPORT_DIR)}}/cpp-coverage.info \ - -o {{quote(REPORT_DIR)}}/merged.info - genhtml {{quote(REPORT_DIR)}}/merged.info -o {{quote(OUTPUT_DIR)}} \ + -a {{ quote(REPORT_DIR) }}/python-coverage.info \ + -a {{ quote(REPORT_DIR) }}/cpp-coverage-through-python-bindings.info \ + -a {{ quote(REPORT_DIR) }}/cpp-coverage.info \ + -o {{ quote(REPORT_DIR) }}/merged.info + genhtml {{ quote(REPORT_DIR) }}/merged.info -o {{ quote(OUTPUT_DIR) }} \ --legend --title "monoprop coverage" \ --prefix {{ project_source_dir }} \ --ignore-errors inconsistent \ @@ -275,8 +370,7 @@ bench-mpi LABEL RANKS *MPIARGS: # Rebuild monoprop with MPI enabled (editable). Run once before `just bench-mpi`. bench-build-mpi: - monoprop_ENABLE_MPI=ON \ - uv sync --all-extras --group bench --reinstall-package monoprop --no-cache -v + monoprop_ENABLE_MPI=ON {{ uv_sync }} --group bench --reinstall-package monoprop --no-cache # Quick sanity run: tiny sizes, skip the slow static benchmarks. bench-smoke: @@ -339,7 +433,7 @@ build-docs: docs-install gen-api doctest-py doctest-docs gen-notebooks # Check exported HTML links (including external URLs). check-doc-links: - lychee --config .lychee.postbuild.toml --root-dir "{{ project_source_dir }}/docs/out" --fallback-extensions html --index-files index.html 'docs/out/**/*.html' + lychee --config .lychee.postbuild.toml 'docs/out/**/*.html' # Serve the documentation locally with hot reloading. serve-docs: diff --git a/prek.toml b/prek.toml index 3ea97f2f..b20664e7 100644 --- a/prek.toml +++ b/prek.toml @@ -87,6 +87,14 @@ hooks = [ ] }, ] +[[repos]] +repo = "local" +hooks = [ + { id = "check-workflow-commands", name = "check workflows call recipes", language = "python", entry = "python tools/check-workflow-commands.py", files = "^\\.github/workflows/.*\\.ya?ml$", pass_filenames = true, additional_dependencies = [ + "PyYAML", + ] }, +] + [[repos]] repo = "https://github.com/python-jsonschema/check-jsonschema" rev = "0.37.4" diff --git a/tools/check-workflow-commands.py b/tools/check-workflow-commands.py new file mode 100755 index 00000000..2fb300f6 --- /dev/null +++ b/tools/check-workflow-commands.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fail when a workflow step runs a command that belongs in the justfile. + +Workflows choose matrices, environment and artifacts; the commands themselves live in +recipes, so that a failing lane can be reproduced locally by running what it ran. This +guards that split, which nothing else can: a copy pasted back into a `run:` block works +perfectly well until it drifts from the recipe it duplicates. + +The exceptions in ``ALLOWED`` are keyed by step name, so renaming an exempt step fails +here and forces the exception to be reconsidered rather than inherited silently. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +import yaml + +# Substrings are matched case-sensitively against the whole `run:` block. Each maps to +# what to do instead. +FORBIDDEN: dict[str, str] = { + "uv sync": "build with `just build [uv sync args…]`", + "uv run": "call the recipe that owns the command", + "uv build": "add a recipe for the distribution being built", + "uv tool run": "add a recipe for the tool being run", + "pytest": "use `just test-py` / `just test-py-mpi`", + "ctest": "use `just test-cpp` / `just test-cpp-mpi`", + "mpiexec": "use the MPI recipe for the leg being run", + "mpirun": "use the MPI recipe for the leg being run", + "gcovr": "use `just code-coverage-collect` / `just code-coverage-aggregate`", + "coverage run": "use `just code-coverage-collect`", + "cmake ": "use the recipe that configures or builds that tree", + "apt-get": "add the package to tools/packages/ and let .github/actions/setup install it", + "brew install": "add the package to tools/packages/ and let .github/actions/setup install it", + "run-clang-tidy": "add a clang-tidy recipe", + "lychee": "use `just check-doc-links`", +} + +# (workflow file name, step name) -> why the step may keep its commands. +ALLOWED: dict[tuple[str, str], str] = { + ( + "qa-analysis.yml", + "Run clang-tidy via run-clang-tidy", + ): "the run-clang-tidy invocation was deliberately left in the workflow", + ( + "deploy.yml", + "Build SDist", + ): "release packaging was deliberately left in the workflow", + ( + "deploy.yml", + "Build sdist and wheel", + ): "release packaging was deliberately left in the workflow", +} + +# Version and help queries are not builds. +BENIGN = re.compile(r"\b(uv|cmake|ctest|lychee)\s+--(version|help)\b") + + +def steps_of(document: object) -> list[tuple[str, str]]: + """Return every (step name, run block) pair in a workflow document.""" + jobs = document.get("jobs") if isinstance(document, dict) else None + if not isinstance(jobs, dict): + return [] + + found: list[tuple[str, str]] = [] + for job in jobs.values(): + if not isinstance(job, dict): + continue + for step in job.get("steps") or []: + if not isinstance(step, dict): + continue + run = step.get("run") + if isinstance(run, str): + found.append((str(step.get("name", "")), run)) + return found + + +def check(path: Path) -> list[str]: + """Return one message per forbidden command found in ``path``.""" + document = yaml.safe_load(path.read_text()) + steps = steps_of(document) + problems = [] + for name, run in steps: + if (path.name, name) in ALLOWED: + continue + body = BENIGN.sub("", run) + for command, remedy in FORBIDDEN.items(): + if command in body: + problems.append( + f"{path}: step '{name}' runs '{command.strip()}' — {remedy}" + ) + + # An exception that no longer matches a step is stale: the step was renamed or the + # commands were moved into a recipe after all. + names = {name for name, _ in steps} + problems.extend( + f"{path}: no step named '{step}'; drop that entry from ALLOWED" + for file_name, step in ALLOWED + if file_name == path.name and step not in names + ) + return problems + + +def main() -> int: + """Check every workflow named on the command line.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("files", nargs="*", type=Path) + args = parser.parse_args() + + problems = [message for path in args.files for message in check(path)] + if not problems: + return 0 + + sys.stderr.write("\n".join(problems) + "\n") + sys.stderr.write( + "\nCommands belong in the justfile; workflows choose matrices, environment and " + "artifacts. Add the exception to ALLOWED in tools/check-workflow-commands.py if a " + "step genuinely cannot use a recipe.\n", + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/packages/apt-mpi.txt b/tools/packages/apt-mpi.txt new file mode 100644 index 00000000..d60c9864 --- /dev/null +++ b/tools/packages/apt-mpi.txt @@ -0,0 +1,4 @@ +# Added to tools/packages/apt.txt when an MPI runtime is requested. + +libopenmpi-dev # MPI::MPI_CXX, and the headers mpi4py builds against +openmpi-bin # mpiexec, for the multi-rank test and coverage runs diff --git a/tools/packages/apt.txt b/tools/packages/apt.txt new file mode 100644 index 00000000..ca01bbd3 --- /dev/null +++ b/tools/packages/apt.txt @@ -0,0 +1,6 @@ +# Debian/Ubuntu packages required to build monoprop from source. + +libboost-dev # Boost.Unordered and friends, a PUBLIC dependency +libboost-test-dev # Boost.Test, the C++ unit-test framework +libmsgpack-cxx-dev # reads the tests/data/*.msgpack fixtures +libhwloc-dev # CpuTopology.cpp: core discovery and partition pinning diff --git a/tools/packages/brew-mpi.txt b/tools/packages/brew-mpi.txt new file mode 100644 index 00000000..f6487b6c --- /dev/null +++ b/tools/packages/brew-mpi.txt @@ -0,0 +1,3 @@ +# Homebrew equivalent of tools/packages/apt-mpi.txt. + +open-mpi # MPI::MPI_CXX, mpiexec, and the mpi4py build headers diff --git a/tools/packages/brew.txt b/tools/packages/brew.txt new file mode 100644 index 00000000..6b0738ac --- /dev/null +++ b/tools/packages/brew.txt @@ -0,0 +1,5 @@ +# Homebrew equivalents of tools/packages/apt.txt. See that file for the format. + +boost # Boost.Unordered and Boost.Test +msgpack-cxx # reads the tests/data/*.msgpack fixtures +hwloc # CpuTopology.cpp: core discovery and partition pinning