From cbb2821a20612805ada37a2cbb6b7d8879e1c6d6 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sat, 29 Aug 2026 19:44:02 +0100 Subject: [PATCH 1/6] chore(benches): make the benchmark set a fixed, runnable table `benches/` carried three benchmarks nobody asked for and four measures where three are read, while the benchmark set that actually matters -- the 10M/100M/1B ladder and the strong/weak scaling rungs -- existed only as flags in a private shell script, so there was no fixed set a PR author could run and report against. Trim to four operations, uniform across both bench modules, both pictures and both fixed models: `build_graph`, `propagate`, `energy`, `gradient`. Drop `test_random_pare`, `test_random_inplace` (which is `propagate` followed by `energy`) and `test_model` (`test_model_propagate` plus a trailing `expectation_value`). These are deletions, never renames: a benchmark name is Bencher's history key. Drop `resting-memory` as a tracked measure, leaving latency, peak memory and an exact-match term count. The raw `memrest`/`membase` recordings stay in `conftest.py` -- unrendered here, but read by the out-of-tree A/B harness. Add `benches/rungs.toml`: one row per cell of the benchmark set, giving the picture, model, operations, geometry, size knobs, the exact term count that configuration produces, and what one rep last cost in wall seconds and GiB per node. `monoprop-bench-rung` runs one rep; `monoprop-bench-ladder` collates them into a markdown block -- the timings, the peak memory, and the resolved parameters of every problem measured -- to paste into a pull request. Nothing runs automatically and nothing gates. Running the rungs a change could plausibly move, and putting the numbers in the PR, is the author's job. The runner refuses a result whose term count, node count, rank count, partition count or round count disagrees with the row, so a mistyped knob fails the cell instead of quietly measuring a different problem. A row nobody has calibrated says so twice -- `expect_terms = 0` and `TBD` on the unmeasured knob -- and refuses to run; `TBD` rather than `0` because `lower_atol = 0` prunes nothing and is the largest problem the model can pose. `cost_seconds` and `cost_gib_per_node` are the opposite: documentation, never a gate. `propagate` carries the full ladder to a billion terms; `build_graph`, `energy` and `gradient` stop at one node, because `build_graph` extends the graph rather than replacing it and retains one layer-set per gate. `bench.yml` loses the `resting-memory` threshold and gains an artifact upload, so a failed Bencher upload no longer loses the measurement. No other CI change. Also fixes a pre-existing bug in `report.py`, where every `test_model_*[hubbard]` and `test_model_*[pauli]` row rendered under the same label. Assisted-by: Claude Opus 5 --- .github/workflows/bench.yml | 15 +- AGENTS.md | 24 +- benches/README.md | 40 +- benches/bench_models.py | 43 - benches/bench_random.py | 39 - benches/conftest.py | 45 +- benches/results/README.md | 8 + benches/rungs.toml | 1048 +++++++++++++++++ docs/content/docs/benchmarks.mdx | 205 +++- packages/monoprop-bench-tools/README.md | 11 +- packages/monoprop-bench-tools/pyproject.toml | 2 + .../src/monoprop_bench_tools/bmf.py | 23 +- .../src/monoprop_bench_tools/report.py | 26 +- .../src/monoprop_bench_tools/rungs.py | 576 +++++++++ .../monoprop-bench-tools/tests/test_bmf.py | 12 +- .../monoprop-bench-tools/tests/test_report.py | 41 +- .../monoprop-bench-tools/tests/test_rungs.py | 368 ++++++ 17 files changed, 2320 insertions(+), 206 deletions(-) create mode 100644 benches/rungs.toml create mode 100644 packages/monoprop-bench-tools/src/monoprop_bench_tools/rungs.py create mode 100644 packages/monoprop-bench-tools/tests/test_rungs.py diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index cd7599c0..befcac60 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -78,6 +78,16 @@ jobs: run: | just bench-bmf "$BENCHER_LABEL" > bmf.json + - name: Upload the raw measurement + if: always() + uses: actions/upload-artifact@v7 + with: + name: bench-${{ inputs.label }} + path: | + bmf.json + benches/results/*.json + if-no-files-found: warn + - uses: bencherdev/bencher@v0.6.12 # The project slug and API key go through the environment rather than the @@ -105,11 +115,6 @@ jobs: --threshold-max-sample-size 64 \ --threshold-lower-boundary _ \ --threshold-upper-boundary 0.10 \ - --threshold-measure resting-memory \ - --threshold-test percentage \ - --threshold-max-sample-size 64 \ - --threshold-lower-boundary _ \ - --threshold-upper-boundary 0.10 \ --threshold-measure terms \ --threshold-test percentage \ --threshold-max-sample-size 1 \ diff --git a/AGENTS.md b/AGENTS.md index cf2f2fd0..a3701d05 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,16 +48,19 @@ The root `pyproject.toml` declares a `[tool.uv.workspace]`, so one `uv.lock` cov - `monoprop` (repository root, `src/monoprop/`) — the library, built by scikit-build-core. - `packages/monoprop-bench-tools/` (`monoprop_bench_tools`) — the reusable half of the benchmark harness, published to PyPI: `memory.cpu` / `memory.gpu` (peak-footprint measurement), `models` - (the benchmarked problem builders), `report` and `bmf` (the artifact renderers). Pure Python, - built by hatchling, versioned off the same git tags as `monoprop`. It ships the console scripts - `monoprop-bench-report` and `monoprop-bench-bmf`. It contains **no benchmarks**. + (the benchmarked problem builders), `report` and `bmf` (the artifact renderers), and `rungs` + (the rung-table loader, runner and ladder collator). Pure Python, built by hatchling, versioned + off the same git tags as `monoprop`. It ships the console scripts `monoprop-bench-report`, + `monoprop-bench-bmf`, `monoprop-bench-rung` and `monoprop-bench-ladder`. It contains **no + benchmarks and no rung table** — those are data whose names campaigns already depend on. - `packages/bench-third-party/` — cross-engine comparison scripts. Listed in the workspace `exclude`: it pins a narrower `requires-python`, a git dependency and linux-x86_64-only CUDA wheels, so it is a standalone uv project with its own `uv.lock`. Run it with `cd packages/bench-third-party && uv sync`, never from the root environment. -- `benches/` — monoprop's own benchmark suite (`conftest.py`, `bench_*.py`, `results/`). It stays in - the repository and imports the tools package. Benchmark names are Bencher's history key, so they - must not move with a library release; that is why the suite is not in `monoprop-bench-tools`. +- `benches/` — monoprop's own benchmark suite (`conftest.py`, `bench_*.py`, `rungs.toml`, + `results/`). It stays in the repository and imports the tools package. Benchmark + names are Bencher's history key and rung ids name artifacts already written, so neither may move + with a library release; that is why the suite is not in `monoprop-bench-tools`. Dependency groups follow from that split: `test` is monoprop's own suite only (cibuildwheel installs it against a built wheel, so it must not reference a workspace member), `workspace-test` adds @@ -83,7 +86,14 @@ Key files: continuous-benchmarking workflow. Both read the schema written by `benches/conftest.py`, so a change to the recorded sections has to land on both sides of the package boundary. Benchmark names are Bencher's history key, so renaming or moving a `bench_*` test orphans its tracked - series. + series. Both bench modules measure the same four operations — `build_graph`, `propagate`, + `energy`, `gradient`. +- `benches/rungs.toml` and `monoprop_bench_tools.rungs`: the benchmark set at the sizes the + library is used at, as a table rather than a loop, so a campaign cannot quietly run a different + grid. Every row carries `expect_terms`, and a result missing it by >0.1% is refused: at a fixed + seed and tolerance the term count is reproducible to the digit, so the gate catches a mistyped + knob rather than tolerating noise. A row with `expect_terms = 0` has never been calibrated and + refuses to run. The scheduler that submits a rung is deliberately not in this repository. - **`rounds > 1` overlaps two rounds' live memory** (`setup=` runs before the prior round's teardown) — pin `--bench-rounds=1`; `record_memory` measures that construction transient, not per-op cost (use `op_memory`). diff --git a/benches/README.md b/benches/README.md index 713a33d4..60459932 100644 --- a/benches/README.md +++ b/benches/README.md @@ -11,20 +11,54 @@ documentation for detailed instructions. ## What lives where This directory holds only monoprop's own benchmarks — `conftest.py` (the fixtures -and the results schema), `bench_random.py`, `bench_models.py`, and `results/`. +and the results schema), `bench_random.py`, `bench_models.py`, `rungs.toml` (the +benchmark set) and `results/`. + +Both bench modules measure the same four operations — `build_graph`, `propagate`, +`energy` and `gradient` — so a number means the same thing whichever problem +produced it. Benchmark names are the key [Bencher](https://bencher.dev/) stores history under, so they stay here rather than moving with a library release. Everything reusable is in the `monoprop-bench-tools` package ([`../packages/monoprop-bench-tools`](../packages/monoprop-bench-tools)): the -memory instrumentation, the model builders, and the two renderers that turn a -run's artifacts into `REPORT.md` and Bencher Metric Format JSON. +memory instrumentation, the model builders, the two renderers that turn a run's +artifacts into `REPORT.md` and Bencher Metric Format JSON, and the runner that +executes one row of `rungs.toml` and gates the result on its term count. Cross-engine comparisons against other propagation libraries live in [`../packages/bench-third-party`](../packages/bench-third-party), a standalone uv project with its own lockfile. +## The rung ladder + +`rungs.toml` is the benchmark set: one row per cell, giving the picture, the model, +the operations, the geometry, the size knobs and the exact term count that +configuration produces. A row, not a loop, so a campaign cannot quietly run a +different grid and two campaigns' numbers are comparable row for row. + +```bash +monoprop-bench-rung benches/rungs.toml list # the set, and what each costs +monoprop-bench-rung benches/rungs.toml --dry-run # the plan, no allocation spent +monoprop-bench-rung benches/rungs.toml --rep 1 # one rep +monoprop-bench-ladder benches/rungs.toml benches/results # the block to paste into the PR +``` + +Nothing runs these for you. Run the rungs your change could plausibly move and put +the block in the pull request: it carries the timings, the peak memory, and the +resolved parameters of every problem measured. + +`expect_terms` is a gate, not documentation: a result missing it by more than 0.1% +is refused, so a mistyped tolerance fails the cell instead of measuring a different +problem under the right name. A row nobody has calibrated says so twice -- +`expect_terms = 0` and `TBD` on the unmeasured knob -- and refuses to run. + +`cost_seconds` and `cost_gib_per_node` are the opposite: what one rep last cost, so +you can see what a rung takes before you spend it. Documentation, never a gate. +The machine and scheduler that run a rung are not in this repository; the table, +the runner and the gate are. + ## Fixed-model sizing At the default `--hubbard-lower-atol 1e-4`, both nominal Hubbard size axes are diff --git a/benches/bench_models.py b/benches/bench_models.py index 7a702335..a5df09c5 100644 --- a/benches/bench_models.py +++ b/benches/bench_models.py @@ -25,10 +25,8 @@ from __future__ import annotations import os -from typing import Any import pytest -from monoprop_bench_tools.memory.cpu import resting_rss_bytes from monoprop_bench_tools.models import MODELS, barrier_setup, barriered # `build_graph` extends the graph, so a driver that re-applies its circuit retains one layer-set per step @@ -45,47 +43,6 @@ def skip_if_graph_will_not_fit(model: str, steps: int) -> None: ) -@pytest.mark.slow -@pytest.mark.parametrize("model", list(MODELS)) -def test_model( - benchmark, - bench_comm, - model_configs, - model, - record_model_config, - record_model_stats, -): - """Benchmark a fixed in-place model simulation (Heisenberg picture).""" - _config_cls, build_fn, steps_fn = MODELS[model] - config = model_configs[model] - steps = steps_fn(config) - record_model_config(model, config) - - state: dict[str, Any] = {} - - def setup(): - state["baseline_rss"] = resting_rss_bytes() - state["built"] = build_fn(config, comm=bench_comm) - return (state["built"], steps), {} - - def run(built, n_steps): - propagator, circuit = built - for _ in range(n_steps): - propagator.propagate(circuit) - return propagator.expectation_value() - - result = benchmark.pedantic( - barriered(run, bench_comm), - setup=barrier_setup(bench_comm, setup), - rounds=1, - iterations=1, - ) - assert isinstance(result, float) - - propagator, _circuit = state["built"] - record_model_stats(model, propagator, state["baseline_rss"]) - - @pytest.mark.slow @pytest.mark.parametrize("model", list(MODELS)) def test_model_build_graph( diff --git a/benches/bench_random.py b/benches/bench_random.py index 31f5e75d..5c00b820 100644 --- a/benches/bench_random.py +++ b/benches/bench_random.py @@ -18,9 +18,6 @@ from monoprop_bench_tools.models import barrier_setup, barriered -PARE_THRESHOLD = 1e-10 -INPLACE_LOWER_ATOL = 1e-5 - def test_random_build_graph( benchmark, @@ -85,22 +82,6 @@ def run(built): assert record_opsize(last[0]) > 0 -def test_random_pare(benchmark, built_graph, bench_comm, bench_rounds): - """Benchmark paring the graph.""" - - def pare(): - return built_graph.expectation_value_and_gradient_functional( - pare_threshold=PARE_THRESHOLD, - ) - - benchmark.pedantic( - barriered(pare, bench_comm), - setup=barrier_setup(bench_comm), - rounds=bench_rounds, - iterations=1, - ) - - def test_random_energy( benchmark, built_graph, random_problem, bench_comm, bench_rounds, op_memory ): @@ -140,23 +121,3 @@ def setup(): ) op_memory.close(built_graph) assert len(gradient) == len(random_problem.parameters) - - -def test_random_inplace(benchmark, make_random_propagator, bench_comm, bench_rounds): - """Benchmark in-place evolution + expectation value (no graph stored).""" - - def setup(): - return (make_random_propagator(lower_atol=INPLACE_LOWER_ATOL),), {} - - def run(built): - propagator, circuit = built - propagator.propagate(circuit) - return propagator.expectation_value() - - result = benchmark.pedantic( - barriered(run, bench_comm), - setup=barrier_setup(bench_comm, setup), - rounds=bench_rounds, - iterations=1, - ) - assert isinstance(result, float) diff --git a/benches/conftest.py b/benches/conftest.py index e6284a69..b4bd6b4f 100644 --- a/benches/conftest.py +++ b/benches/conftest.py @@ -45,11 +45,7 @@ import psutil import pytest -from monoprop_bench_tools.memory.cpu import ( - HighWaterMark, - pinned_thread_summary, - resting_rss_bytes, -) +from monoprop_bench_tools.memory.cpu import HighWaterMark, pinned_thread_summary from monoprop_bench_tools.models import ( MODELS, RandomProblem, @@ -143,8 +139,6 @@ def _spread(comm: Any, value: int) -> dict[str, int]: "memhwm": {}, # node id -> summed peak RSS, whole test, setup() included "memhwm_max": {}, # node id -> worst-rank peak RSS, whole test, setup() included "opsize": {}, # picture / model / node id -> {"terms": n} - "memrest": {}, # picture / model -> resting RSS bytes - "membase": {}, # fixed model -> resting RSS bytes before the model is built "configs": {}, # fixed model -> config dataclass fields "opmem": {}, # fixed model -> per-field operator memory split (bytes) # Timed call only (see ``OpMemory``), each {"sum", "max"}. @@ -348,10 +342,8 @@ def _do(model: str, config: Any) -> None: return _do -def _record_model_stats( - comm: Any, key: str, propagator: Any, baseline_rss: int | None = None -) -> None: - """Record term count, operator memory breakdown and footprint under ``key``.""" +def _record_model_stats(comm: Any, key: str, propagator: Any) -> None: + """Record term count and operator memory breakdown under ``key``.""" _record("opsize", key, {"terms": _reduce_sum(comm, propagator.size())}) # Placement is only observable while the propagator's threads are alive. @@ -366,25 +358,6 @@ def _record_model_stats( {k: _reduce_sum(comm, v) for k, v in breakdown().items()}, ) - resting = _reduce_sum(comm, resting_rss_bytes()) - if resting: # 0 => /proc unavailable; skip rather than record 0 MiB - _record("memrest", key, resting) - - if baseline_rss is not None: - baseline = _reduce_sum(comm, baseline_rss) - if baseline: - _record("membase", key, baseline) - - -@pytest.fixture -def record_model_stats(bench_comm: Any) -> Callable[..., None]: - """Return ``record(model, propagator, baseline_rss)`` for fixed-model runs.""" - - def _do(model: str, propagator: Any, baseline_rss: int) -> None: - _record_model_stats(bench_comm, model, propagator, baseline_rss) - - return _do - class OpMemory: """Records one benchmarked operation's memory over the timed call alone. @@ -541,10 +514,8 @@ def built_graph( """Return a propagator whose graph has been built (no coefficients contracted). Session-scoped per picture so the graph is built once and shared across the - read-only graph benchmarks (``pare``, ``energy``, ``gradient``). - - Also records the operator size and resting footprint for this picture while the - graph is resident. + read-only graph benchmarks (``energy``, ``gradient``), and records the operator + size for this picture while the graph is resident. """ mp, circuit = build_random_propagator( random_problem, comm=bench_comm, schrodinger=picture == "schrodinger" @@ -554,12 +525,6 @@ def built_graph( # Under MPI the operator is partitioned, so sum the partitions. _record("opsize", picture, {"terms": _reduce_sum(bench_comm, mp.size())}) - # Settled RSS once the build's transients are released -- the persistent - # footprint the per-operation peak cannot see. - resting = _reduce_sum(bench_comm, resting_rss_bytes()) - if resting: # 0 => /proc unavailable; skip rather than record 0 MiB - _record("memrest", picture, resting) - return mp diff --git a/benches/results/README.md b/benches/results/README.md index 39736725..ac3c274c 100644 --- a/benches/results/README.md +++ b/benches/results/README.md @@ -8,3 +8,11 @@ produced and combined. These files should **NOT** be added to the repo — only this `README.md` is tracked (see the `benches/results/**` rule in the top-level `.gitignore`). + +## Sections nothing here renders + +`