Separate benchmark timing from routine CI

.github/actions/setup-tools/action.yml

@@ -29,7 +29,7 @@ runs:
       shell: bash
       env:
         RESEARCH_REPO_TOOLS_HOME: ${{ runner.temp }}/rrt
-      run: uv run --locked --managed-python --only-group tooling python tooling/shared_tooling.py setup
+      run: uv run --locked --managed-python --only-group tooling research-repo-tools setup
 
     - name: Export verified managed paths
       shell: bash

.github/workflows/benchmarks.yml

@@ -1,69 +1,22 @@
 name: Benchmarks
 
-# Detect performance regressions.
-# - Push to main: run benchmarks, save Criterion baseline, upload as artifact.
-# - PRs: find latest main baseline artifact, download, compare, report.
-# Regressions are warning-only — this workflow never fails on regression.
-
-permissions:
-  contents: read
-  actions: read
-
+# Exploratory timings for a maintainer-selected ref.
 on:
-  push:
-    branches:
-      - main
-    paths:
-      - "src/**"
-      - "benches/**"
-      - "tests/exact_bench_config.rs"
-      - ".config/nextest.toml"
-      - "Cargo.toml"
-      - "Cargo.lock"
-      - "justfile"
-      - "rust-toolchain.toml"
-      - "pyproject.toml"
-      - "uv.lock"
-      - ".python-version"
-      - ".github/actions/setup-tools/action.yml"
-      - ".github/workflows/benchmarks.yml"
-  pull_request:
-    branches:
-      - main
-    paths:
-      - "src/**"
-      - "benches/**"
-      - "tests/exact_bench_config.rs"
-      - ".config/nextest.toml"
-      - "Cargo.toml"
-      - "Cargo.lock"
-      - "justfile"
-      - "rust-toolchain.toml"
-      - "pyproject.toml"
-      - "uv.lock"
-      - ".python-version"
-      - ".github/actions/setup-tools/action.yml"
-      - ".github/workflows/benchmarks.yml"
   workflow_dispatch:
 
-concurrency:
-  group: >
-    bench-${{ github.workflow }}-${{
-      github.event_name == 'pull_request' &&
-      github.event.pull_request.number ||
-      github.ref
-    }}
-  cancel-in-progress: true
+permissions:
+  contents: read
 
 env:
   CARGO_TERM_COLOR: always
+  CRITERION_HOME: target/criterion
   RUST_BACKTRACE: 1
 
 jobs:
   bench:
     runs-on: ubuntu-latest
-    # The full exact suite has 260+ cases: Criterion's default 3 s warm-up
-    # plus 5 s measurement already needs 35+ minutes, before build/analysis.
+    # At Criterion's default 8 s of warm-up and measurement per case,
+    # 400+ cases need more than 53 minutes before build and analysis.
     timeout-minutes: 90
 
     steps:
@@ -77,157 +30,13 @@ jobs:
       - name: Validate benchmark inputs
         run: just test-bench-inputs
 
-      # ── PR: find and download the latest main baseline ──────────────
-      - name: Find latest main baseline
-        if: github.event_name == 'pull_request'
-        id: find-baseline
-        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
-        with:
-          script: |
-            // Find the most recent successful run of this workflow on main.
-            const runs = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              workflow_id: 'benchmarks.yml',
-              branch: 'main',
-              status: 'success',
-              per_page: 5,
-            });
-
-            for (const run of runs.data.workflow_runs) {
-              const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                run_id: run.id,
-              });
-
-              const baseline = artifacts.data.artifacts.find(
-                a => a.name === 'bench-baseline-main' && !a.expired
-              );
-
-              if (baseline) {
-                console.log(`Found baseline from run ${run.id} (${run.created_at})`);
-                core.setOutput('found', 'true');
-                core.setOutput('run_id', run.id.toString());
-                return;
-              }
-            }
-
-            console.log('No baseline artifact found');
-            core.setOutput('found', 'false');
+      - name: Measure the full exact suite
+        run: just bench-exact --noplot
 
-      - name: Download baseline artifact
-        if: >
-          github.event_name == 'pull_request' &&
-          steps.find-baseline.outputs.found == 'true'
-        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
-        with:
-          name: bench-baseline-main
-          path: target/criterion
-          run-id: ${{ steps.find-baseline.outputs.run_id }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      # ── PR: run benchmarks and compare ──────────────────────────────
-      - name: Run benchmarks (compare against main)
-        if: github.event_name == 'pull_request'
-        id: bench-compare
-        run: |
-          set -euo pipefail
-
-          if [ -d target/criterion/exact_d2/det/main ]; then
-            echo "::notice::Baseline found — comparing against main"
-            echo "comparison_available=true" >> "$GITHUB_OUTPUT"
-            # --baseline-lenient rather than --baseline: benches added on the
-            # PR branch that don't yet exist in the main baseline run without
-            # a comparison instead of aborting the whole run.
-            # Keep normal sampling; plots/HTML are unused by the summary.
-            cargo bench --locked --features bench,exact --bench exact \
-              -- --noplot --baseline-lenient main 2>&1 | tee bench-output.txt
-          else
-            echo "::notice::No baseline found — running without comparison"
-            echo "comparison_available=false" >> "$GITHUB_OUTPUT"
-            cargo bench --locked --features bench,exact --bench exact \
-              -- --noplot 2>&1 | tee bench-output.txt
-          fi
-
-          if grep -q "Performance has regressed" bench-output.txt; then
-            echo "regression=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "regression=false" >> "$GITHUB_OUTPUT"
-          fi
-
-      # ── Main push: run benchmarks and save baseline ─────────────────
-      - name: Run benchmarks (save baseline)
-        if: >
-          (github.event_name == 'push' || github.event_name == 'workflow_dispatch') &&
-          github.ref == 'refs/heads/main'
-        run: >
-          cargo bench --locked --features bench,exact --bench exact
-          -- --noplot --save-baseline main
-
-      - name: Run benchmarks (manual ref)
-        if: github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main'
-        run: cargo bench --locked --features bench,exact --bench exact -- --noplot
-
-      - name: Upload baseline artifact
-        if: >
-          (github.event_name == 'push' || github.event_name == 'workflow_dispatch') &&
-          github.ref == 'refs/heads/main'
+      - name: Upload exploratory timings
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: bench-baseline-main
-          path: target/criterion
-          retention-days: 90
+          name: exploratory-exact-${{ github.run_id }}-${{ github.run_attempt }}
+          path: target/criterion/
+          retention-days: 14
           if-no-files-found: error
-
-      # ── PR: report results ──────────────────────────────────────────
-      - name: Benchmark summary
-        if: github.event_name == 'pull_request' && always()
-        run: |
-          set -euo pipefail
-
-          comparison_available="${BENCH_COMPARISON_AVAILABLE:-}"
-          regression="${BENCH_REGRESSION:-}"
-
-          # Lenient comparisons silently skip missing baselines. Require a
-          # change report for every analyzed case before claiming coverage.
-          if [ -f bench-output.txt ]; then
-            analyzed="$(grep -c '^Benchmarking .*: Analyzing$' bench-output.txt || true)"
-            compared="$(grep -c '^[[:space:]]*change:' bench-output.txt || true)"
-            if [ "$analyzed" = "0" ] || [ "$analyzed" != "$compared" ]; then
-              comparison_available=false
-            fi
-          else
-            comparison_available=false
-          fi
-
-          if [ "$comparison_available" != "true" ] || [ -z "$regression" ]; then
-            {
-              echo "### ❓ Benchmark Comparison Unavailable"
-              echo ""
-              echo "No usable comparison against the main baseline was produced."
-              echo "Benchmarks may be incomplete or lack a baseline; no regression claim can be made."
-            } >> "$GITHUB_STEP_SUMMARY"
-            echo "::warning::Benchmark comparison unavailable"
-          elif [ "$regression" = "true" ]; then
-            {
-              echo "### ⚠️ Performance Regression Detected"
-              echo ""
-              echo "Exact-arithmetic benchmarks show regression vs main baseline."
-              echo "This is a **warning only** — the workflow will not fail."
-              echo ""
-              echo '```'
-              grep -B1 "Performance has regressed" bench-output.txt || true
-              echo '```'
-            } >> "$GITHUB_STEP_SUMMARY"
-            echo "::warning::Performance regression detected in exact-arithmetic benchmarks"
-          else
-            {
-              echo "### ✅ No Performance Regression"
-              echo ""
-              echo "Exact-arithmetic benchmarks are within expected range vs main."
-            } >> "$GITHUB_STEP_SUMMARY"
-          fi
-        env:
-          BENCH_COMPARISON_AVAILABLE: ${{ steps.bench-compare.outputs.comparison_available }}
-          BENCH_REGRESSION: ${{ steps.bench-compare.outputs.regression }}

.github/workflows/ci.yml

@@ -91,7 +91,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          uvx --python 3.15 --from research-repo-tools==0.2.4 python -I - <<'PY'
+          uvx --python 3.15 --from research-repo-tools==0.2.6 python -I - <<'PY'
           import os
           import sys
           from pathlib import Path

.github/workflows/release-benchmarks.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Install trusted package runner
         uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
         with:
-          version: "0.13.6"
+          version: "0.13.7"
           enable-cache: false
           cache-python: false
 
@@ -54,7 +54,7 @@ jobs:
             echo "::error::Dispatch with --ref $RELEASE_TAG and the matching tag input"
             exit 1
           fi
-          target="$(uvx --python 3.15 --from research-repo-tools==0.2.4 research-repo-tools \
+          target="$(uvx --python 3.15 --from research-repo-tools==0.2.6 research-repo-tools \
             --root "$RUNNER_TEMP" performance release-draft "$GH_REPO" "$RELEASE_TAG" \
             --commit "$GITHUB_SHA" --expected-title "$RELEASE_TAG" | jq -c .)"
           echo "target=$target" >> "$GITHUB_OUTPUT"
@@ -109,7 +109,7 @@ jobs:
           if [[ "$BENCH_SUITE" == public-exact ]]; then
             features=bench,exact
           fi
-          uvx --python 3.15 --from research-repo-tools==0.2.4 research-repo-tools performance pack-suite \
+          uvx --python 3.15 --from research-repo-tools==0.2.6 research-repo-tools performance pack-suite \
             tooling/criterion-saved.toml "$BENCH_SUITE" --saved-sample "$RELEASE_TAG" \
             --output "$output/criterion.tar.gz" --revision "$GITHUB_SHA" \
             --session "$GITHUB_REPOSITORY:$GITHUB_RUN_ID:$GITHUB_RUN_ATTEMPT" \
@@ -179,7 +179,7 @@ jobs:
       - name: Install trusted package runner
         uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
         with:
-          version: "0.13.6"
+          version: "0.13.7"
           enable-cache: false
           cache-python: false
 
@@ -202,7 +202,7 @@ jobs:
 
           asset="la-stack-${RELEASE_TAG}-criterion-baseline.tar.gz"
           cp target/release-suites/public-default/inventory.json target/release-benchmark-inventory.json
-          uvx --python 3.15 --from research-repo-tools==0.2.4 research-repo-tools \
+          uvx --python 3.15 --from research-repo-tools==0.2.6 research-repo-tools \
             --root "$GITHUB_WORKSPACE" performance assemble-suites tooling/criterion-saved.toml \
             target/release-suites/public-default/criterion.tar.gz \
             target/release-suites/public-exact/criterion.tar.gz \
@@ -230,7 +230,7 @@ jobs:
       - name: Install trusted package runner
         uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
         with:
-          version: "0.13.6"
+          version: "0.13.7"
           enable-cache: false
           cache-python: false
 
@@ -251,7 +251,7 @@ jobs:
 
           [[ "$RELEASE_ASSET" == "la-stack-${RELEASE_TAG}-criterion-baseline.tar.gz" ]] || exit 1
           printf '%s\n' "$RELEASE_TARGET" > "$RUNNER_TEMP/release-target.json"
-          uvx --python 3.15 --from research-repo-tools==0.2.4 research-repo-tools \
+          uvx --python 3.15 --from research-repo-tools==0.2.6 research-repo-tools \
             --root "$RUNNER_TEMP" performance release-upload "$GH_REPO" "$RELEASE_TAG" \
             "$GITHUB_WORKSPACE/$RELEASE_ASSET" --target "$RUNNER_TEMP/release-target.json"
 

CONTRIBUTING.md

@@ -10,7 +10,6 @@
 [![Audit dependencies][audit-badge]][audit-workflow]
 [![zizmor][zizmor-badge]][zizmor-workflow]
 [![Repository Rule SARIF][semgrep-badge]][semgrep-workflow]<br>
-[![Benchmarks][benchmarks-badge]][benchmarks-workflow]
 [![Release Benchmarks][release-benchmarks-badge]][release-benchmarks-workflow]
 [![Dependabot approval and auto-merge][dependabot-badge]][dependabot-workflow]
 <!-- markdownlint-enable MD033 -->
@@ -38,7 +37,7 @@ exact version declared in `pyproject.toml`, and `jq`. Authenticate the GitHub CL
 the declared Python, Rust, Cargo tools, and Just through the locked PyPI package:
 
 ```bash
-uv run --locked --managed-python --only-group tooling python tooling/shared_tooling.py setup
+uv run --locked --managed-python --only-group tooling research-repo-tools setup
 ```
 
 Set up the remaining development tools and validate the checkout:
@@ -54,7 +53,7 @@ change files. Run bare `just` for the complete generated command list, including
 arguments and descriptions. The [command policy](docs/dev/docs.md#command-discovery-and-placement)
 owns recipe naming and documentation placement.
 
-Setup, tool checks, updates, changelog, and opt-in CodeRabbit review use the published `research-repo-tools==0.2.4` package,
+Setup, tool checks, updates, changelog, and opt-in CodeRabbit review use the published `research-repo-tools==0.2.6` package,
 locked in the `tooling` dependency group and included by `dev`. Its `python-tools`
 extra owns the pytest, Ruff and Ty versions.
 `toolchain.inherit-python-tools` checks those versions against the lock and
@@ -86,8 +85,8 @@ Then `just update-dependencies` upgrades Cargo requirements and refreshes Cargo'
 lock, resolves exact direct Python `dev` pins, refreshes the complete `uv.lock`,
 and explicitly synchronizes `dev` with managed Rust available for native builds.
 Included groups retain their constraints, including the exact shared-package pin.
-Semgrep scans and fixture checks share the root Python 3.15 environment and lockfile. Preserve its exact
-pin, native-wheel requirement and compatibility overrides for MCP, PyJWT,
+Semgrep scans and fixture checks share the root Python 3.15 environment and lockfile. Preserve its platform
+constraints, native-wheel requirement and compatibility overrides for MCP, PyJWT,
 OpenTelemetry and Python 3.15 Windows wheels. Scanner commands retain the Linux
 stack-limit workaround required by Semgrep on Python 3.15. The hosted dependency
 audit checks the root Python lockfile with the shared OSV adapter; its existing
@@ -253,6 +252,11 @@ human contributor performs and reviews commits, pushes, tags, and releases.
 
 ### Validation Workflow
 
+On PRs and pushes to `main`, native CI compiles the benchmark targets, runs
+the input tests, and exercises the exact harness in Criterion test mode.
+Timing runs are opt-in. See [Benchmarking](docs/BENCHMARKING.md#ci-validation-and-manual-timings)
+for the coverage and dispatch commands.
+
 Start with the smallest test selection that exercises the changed behavior. Use
 `cargo nextest run <test-name>` for a focused runnable test, `cargo test --doc
 <filter>` for a focused doctest, or `cargo nextest run --test <crate>` for one
@@ -267,7 +271,7 @@ For final validation of a non-core change, compose each affected surface once:
 - Rust unit tests: `just test-unit`
 - Doctests: `just test-doc` and, when relevant, `just test-doc-exact`
 - Integration tests: `just test-integration`
-- Benchmark inputs or harnesses: `just test-bench-inputs` or `just bench-compile`
+- Benchmark inputs or harnesses: `just test-bench-inputs`, `just bench-compile`, or `just test-bench-exact`
 - Examples: `just examples`
 
 For individual edited file formats, use these focused checks and formatters:
@@ -441,8 +445,6 @@ Report suspected vulnerabilities privately through the process in
 
 [audit-badge]: https://github.com/acgetchell/la-stack/actions/workflows/audit.yml/badge.svg?branch=main
 [audit-workflow]: https://github.com/acgetchell/la-stack/actions/workflows/audit.yml
-[benchmarks-badge]: https://github.com/acgetchell/la-stack/actions/workflows/benchmarks.yml/badge.svg?branch=main
-[benchmarks-workflow]: https://github.com/acgetchell/la-stack/actions/workflows/benchmarks.yml
 [ci-badge]: https://github.com/acgetchell/la-stack/actions/workflows/ci.yml/badge.svg?branch=main
 [ci-workflow]: https://github.com/acgetchell/la-stack/actions/workflows/ci.yml
 [clippy-badge]: https://github.com/acgetchell/la-stack/actions/workflows/rust-clippy.yml/badge.svg?branch=main