chore(ci): 👷 benchmark the ladder rungs, one testbed per shape and silicon

.github/runs-on.yml

@@ -6,10 +6,8 @@
 runners:
   # Used by .github/workflows/bench_bare_metal.yml.
   #
-  # Every field is pinned rather than ranged. Bencher compares each run against
-  # this testbed's own history, so a different instance type or core count is a
-  # step change in the series rather than a data point -- widening any of these
-  # means retiring the testbed and starting a new one.
+  # Pinned rather than ranged: the testbed is named after this CPU and its core count, so widening
+  # any field renames the testbed and stops the old series rather than corrupting it.
   bench:
     # Avoid expand to partial family
     family: ["c7i.4xlarge"]

.github/workflows/bench.yml

@@ -1,8 +1,6 @@
 name: Benchmark
 
-# Shared body of the Bencher continuous-benchmarking runs. Callers choose the
-# machine and the Bencher testbed; everything downstream is identical, so the
-# testbeds stay comparable in shape even though their absolute numbers are not.
+# Shared body of the Bencher runs; the caller names the machine and the benches/LADDER.md rungs.
 
 on:
   workflow_call:
@@ -11,24 +9,17 @@ on:
         description: Value for the job's `runs-on`.
         required: true
         type: string
-      testbed:
-        description: Bencher testbed. Must change whenever the hardware does.
-        required: true
-        type: string
       label:
-        description: Benchmark label, used to name the result artifacts.
+        description: Result-artifact prefix; each rung uploads as `<label>-<rung>`.
         required: true
         type: string
-      bench-markers:
-        description: pytest marker expression. Empty selects every benchmark.
-        required: false
-        type: string
-        default: not slow
-      bench-rounds:
-        description: Fixed timing rounds per benchmark.
-        required: false
+      rungs:
+        description: >-
+          One rung per line, `<name> <ranks> <partitions> <rounds> | <pytest args>`.
+          `partitions` is an integer or `per-rank` for (physical cores / ranks).
+          Blank lines and `#` comments are ignored.
+        required: true
         type: string
-        default: "5"
 
 env:
   FORCE_COLOR: 3
@@ -41,6 +32,7 @@ jobs:
     # Skips forks, which have no BENCHER_API_KEY.
     if: ${{ github.repository == 'Algorithmiq/monoprop' }}
     runs-on: ${{ inputs.runner }}
+    timeout-minutes: 120
 
     steps:
       - uses: actions/checkout@v7.0.1
@@ -55,61 +47,42 @@ jobs:
       - name: Install package
         env:
           SKBUILD_CMAKE_DEFINE: monoprop_ENABLE_CXX_UNIT_TESTS=OFF
+          monoprop_ENABLE_MPI: "ON"
         run: just build --group bench
 
-      - name: Run benchmarks
-        env:
-          BENCHER_LABEL: ${{ inputs.label }}
-          monoprop_BENCH_MARKERS: ${{ inputs.bench-markers }}
-          monoprop_BENCH_ROUNDS: ${{ inputs.bench-rounds }}
-        run: |
-          just bench-ci "$BENCHER_LABEL"
+      - name: Resolve the core count
+        run: just bench-ci-resolve-cores
 
-      - name: Convert results to Bencher Metric Format
+      - name: Run the ladder rungs
         env:
-          BENCHER_LABEL: ${{ inputs.label }}
-        run: |
-          just bench-bmf "$BENCHER_LABEL" > bmf.json
+          BENCH_LABEL: ${{ inputs.label }}
+          RUNGS: ${{ inputs.rungs }}
+          # Empty, so a rung's `-k` is the only selector and the slow fixed models are reachable.
+          monoprop_BENCH_MARKERS: ""
+          # The RunsOn images run as root, and OpenMPI refuses to launch there without these.
+          OMPI_ALLOW_RUN_AS_ROOT: "1"
+          OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1"
+        run: just bench-ci-rungs "$BENCH_LABEL"
 
       - name: Upload the raw measurement
         if: always()
         uses: actions/upload-artifact@v7
         with:
           name: bench-${{ inputs.label }}
           path: |
-            bmf.json
+            bmf-*.json
             benches/results/*.json
           if-no-files-found: warn
 
       - uses: bencherdev/bencher@v0.6.12
 
-      # The project slug and API key go through the environment rather than the
-      # command line: `bencher` reads BENCHER_PROJECT / BENCHER_API_KEY natively,
-      # and it keeps a maintainer-set variable out of the shell's parsing.
+      # The project slug and API key go through the environment: `bencher` reads
+      # BENCHER_PROJECT / BENCHER_API_KEY natively, and it keeps a maintainer-set
+      # variable out of the shell's parsing.
       - name: Track benchmarks with Bencher
         env:
           BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT || 'monoprop' }}
           BENCHER_API_KEY: ${{ secrets.BENCHER_API_KEY }}
-          BENCHER_TESTBED: ${{ inputs.testbed }}
-        run: |
-          bencher run \
-            --branch main \
-            --adapter json \
-            --file bmf.json \
-            --github-actions "${{ secrets.GITHUB_TOKEN }}" \
-            --thresholds-reset \
-            --threshold-measure latency \
-            --threshold-test t_test \
-            --threshold-max-sample-size 64 \
-            --threshold-lower-boundary _ \
-            --threshold-upper-boundary 0.99 \
-            --threshold-measure peak-memory \
-            --threshold-test percentage \
-            --threshold-max-sample-size 64 \
-            --threshold-lower-boundary _ \
-            --threshold-upper-boundary 0.10 \
-            --threshold-measure terms \
-            --threshold-test percentage \
-            --threshold-max-sample-size 1 \
-            --threshold-lower-boundary 0.0 \
-            --threshold-upper-boundary 0.0
+          BENCHER_BRANCH: ${{ github.ref_name }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: just bench-ci-track

.github/workflows/bench_bare_metal.yml

@@ -1,12 +1,6 @@
 name: Track benchmarks (bare metal)
 
-# The same benchmark profile as `bench_main.yml`, on a dedicated EC2 node
-# provisioned per run by RunsOn. The point is the noise floor: nothing else is
-# scheduled on the machine, so this testbed can eventually carry thresholds
-# tight enough to gate on, which the shared GitHub runner cannot.
-#
-# The node exists only for the duration of the run, so cost tracks the number
-# of commits to main rather than wall-clock time.
+# monoprop's continuous benchmarking, on a dedicated EC2 node provisioned per run by RunsOn.
 
 on:
   workflow_dispatch:
@@ -20,22 +14,18 @@ concurrency:
 
 jobs:
   bare-metal:
-    # Inert until a maintainer sets the BENCH_BARE_METAL repository variable to
-    # 'true'. Without a RunsOn stack behind the label the job would sit queued
-    # until it timed out, once per commit.
+    # Inert until a maintainer sets BENCH_BARE_METAL to 'true': without a RunsOn stack behind the
+    # label the job would queue until it timed out, once per commit.
     if: ${{ vars.BENCH_BARE_METAL == 'true' }}
     uses: ./.github/workflows/bench.yml
     with:
-      # The `runner=bench` shape is defined in .github/runs-on.yml; run_id
-      # namespaces the runner so no other job can claim it.
+      # The `runner=bench` shape is defined in .github/runs-on.yml; run_id namespaces it.
       runner: runs-on=${{ github.run_id }}/runner=bench
-      # Names the hardware, not the provider: a different instance type is a
-      # different testbed, because its history cannot be compared to this one.
-      testbed: aws-c7i-4xlarge
       label: ci-bare-metal
-      # A dedicated node can afford the slow fixed models
-      bench-markers: ""
-      bench-rounds: "3"
+      rungs: |
+        L1  1 1        3 | -k "(test_model_propagate and (hubbard or pauli)) or (test_random_gradient and heisenberg)" --hubbard-cutoff=10 --hubbard-lower-atol=4.2e-05 --pauli-cutoff=12 --pauli-lower-atol=1.22e-04 --num-generators=1000 --num-modes=142 --cutoff=6 --obs-terms=295000
+        L2a 1 per-rank 1 | -k "(test_random_build_graph or test_random_energy or test_random_gradient) and heisenberg" --num-generators=1000 --num-modes=142 --cutoff=6 --obs-terms=2500000
+        L2b 4 per-rank 1 | -k "(test_random_build_graph or test_random_energy or test_random_gradient) and heisenberg" --num-generators=1000 --num-modes=142 --cutoff=6 --obs-terms=2500000
     secrets: inherit
     permissions:
       contents: read

.github/workflows/scripts/check_shape.py

@@ -0,0 +1,60 @@
+# Copyright 2026 Algorithmiq
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Fail a rung that did not run at the shape it was asked for.
+
+A knob that fails to reach the ranks does not fail the run: it measures one partition per
+rank at a plausible wall time. Neither does `partitions` above a rank's visible cores --
+that disables placement entirely, at 13.8x to 24.6x, warning only on C++ stderr, which
+pytest's capture eats.
+
+Usage: check_shape.py <results/<label>.json> <ranks> <partitions>
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+
+def main(argv: list[str]) -> str | None:
+    path, ranks, partitions = Path(argv[1]), int(argv[2]), argv[3]
+    meta = json.loads(path.read_text())["meta"]
+    pin = meta.get("pinning", {})
+    threads = pin.get("threads", 0)
+    placed = pin.get("single_cpu_threads_min", 0)
+
+    print(
+        f"  ranks={meta['ranks']} partitions_env={meta['partitions_env']} "
+        f"threads={meta['monoprop_threads']} pinned>={placed} of {threads} "
+        f"mask={pin.get('affinity_cpus_min')}..{pin.get('affinity_cpus_max')}"
+    )
+
+    if meta["ranks"] != ranks or meta["partitions_env"] != partitions:
+        return (
+            f"::error::{path.stem} ran at ranks={meta['ranks']} "
+            f"partitions_env={meta['partitions_env']}, asked for {ranks}x{partitions}"
+        )
+    # An all-zero summary means /proc was unreadable, not that nothing was pinned.
+    if int(partitions) > 1 and threads and not placed:
+        return (
+            f"::error::{path.stem} placed no thread on a CPU of its own. "
+            "partitions above the rank's visible cores returns an empty placement order."
+        )
+    return None
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

.github/workflows/scripts/cpu_slug.py

@@ -0,0 +1,47 @@
+# Copyright 2026 Algorithmiq
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Print this run's CPU as a Bencher testbed slug, e.g. `intel-xeon-platinum-8488c`.
+
+Naming a series after the silicon rather than a provider's instance label means the name
+changes exactly when the hardware does, which is the property Bencher needs. The brand comes
+from the cpuinfo block pytest-benchmark already writes into `time-<label>.json`.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+NOISE = re.compile(r"\((?:r|tm)\)|\bprocessor\b", re.IGNORECASE)
+SEPARATORS = re.compile(r"[^a-z0-9]+")
+
+
+def slugify(brand: str) -> str:
+    return SEPARATORS.sub("-", NOISE.sub(" ", brand).lower()).strip("-") or "unknown-cpu"
+
+
+def main() -> str | None:
+    paths = sorted(Path("benches/results").glob("time-*.json"))
+    if not paths:
+        return "::error::no timing artifact to read the CPU from"
+    brand = json.loads(paths[0].read_text())["machine_info"]["cpu"]["brand_raw"]
+    print(slugify(str(brand)))
+    return None
+
+
+if __name__ == "__main__":
+    sys.exit(main())

README.md

@@ -4,7 +4,6 @@
 [![Documentation](https://github.com/Algorithmiq/monoprop/actions/workflows/docpages.yml/badge.svg)](https://docs.monoprop.algorithmiq.tech/)
 [![Test monoprop](https://github.com/Algorithmiq/monoprop/actions/workflows/test.yml/badge.svg)](https://github.com/Algorithmiq/monoprop/actions/workflows/test.yml)
 [![codecov](https://codecov.io/gh/Algorithmiq/monoprop/graph/badge.svg?token=1GgmPnNUxk)](https://codecov.io/gh/Algorithmiq/monoprop)
-[![Track benchmarks](https://github.com/Algorithmiq/monoprop/actions/workflows/bench_main.yml/badge.svg)](https://github.com/Algorithmiq/monoprop/actions/workflows/bench_main.yml)
 
 `monoprop` is a high-performance C++ library with Python bindings for **Majorana and
 Pauli propagation** — a backend for classically simulating and variationally