v0.5.24 — Benchmark regression gate for CI (Loop 5)

.github/workflows/ci.yml

@@ -37,6 +37,9 @@ jobs:
       - name: Leakage check (informational)
         run: make bench-leakage
 
+      - name: Benchmark regression gate (AUROC drop <= 0.02 vs committed baseline)
+        run: make bench-gate
+
       - name: Hidden-active benchmark (require EF >= 1.5 at k=5)
         run: |
           PYTHONPATH=src python -c "

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help demo test lint ci clean bench-leakage bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-selectivity bench-feature-decomp generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter
+.PHONY: help demo test lint ci clean bench-leakage bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-selectivity bench-feature-decomp bench-gate generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter
 
 PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
 PYTEST  := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
@@ -32,6 +32,7 @@ help:
 	@echo "  make bench-expert-ablation    Expert composite vs ensemble ablation (honesty check)"
 	@echo "  make bench-selectivity        Within-AMP selectivity (hemolytic vs selective)"
 	@echo "  make bench-feature-decomp     Per-feature selective_vs_hemolytic decomposition"
+	@echo "  make bench-gate               Benchmark regression gate (AUROC drift check)"
 	@echo "  make bench-hidden-active      Hidden-positive recovery on mixed benchmark set"
 	@echo ""
 	@echo "  make wave0-5-gate-check     Run Wave 0.5 gates W0.5-1 through W0.5-7"
@@ -164,6 +165,12 @@ bench-triage:
 	PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench triage \
 		--out outputs/triage_benchmark_report.json
 
+bench-gate:
+	PYTHONPATH=src $(PYTHON) scripts/benchmark_gate.py \
+		--baseline outputs/metrics_snapshot.json \
+		--tolerance 0.02 \
+		--out outputs/bench_gate_report.md
+
 bench-feature-decomp:
 	PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench feature-decomp \
 		--out outputs/feature_decomp_report.json

docs/50_LOOP_PLAN.md

@@ -186,12 +186,12 @@ Phase 4: Not started
 | 2 ✅ | README doesn't document calibration CLI or policy files; no 50-loop strategic plan exists | Updated README with calibration flow and repo map; 50-LOOP_PLAN.md created | 1652 pass, README reviewed |
 | 3 ✅ | Doc audit — verify freshness, deduplicate, archive stale content | Performed as part of Loop 1/2 structure work; calibration docs up to date | Docs consistent |
 | 4 ✅ | No end-to-end smoke test for the full calibration chain (panel→intake→gate→verdict) | `tests/test_calibration_e2e.py`: 14 new tests exercising Python API, CLI, and Makefile targets for both passing and failing paths | 1669 pass, full calibration flow verified |
+| 5 ✅ | No CI benchmark regression gate — AUROC can silently degrade | `scripts/benchmark_gate.py`, `tests/test_benchmark_gate.py` (13 tests), `make bench-gate` target, CI step that fails on AUROC drop >0.02 | 1682 pass, `make bench-gate` exits 0, CI gate committed |
 
 ### Remaining (Phase 0)
 
 | Loop | Bottleneck | Deliverable | Verification |
 |------|-----------|-------------|-------------|
-| 5 | No CI job that runs `make demo` + `make validate-scoring` + `make bench-triage` on every PR | `.github/workflows/ci.yml` expanded with benchmark gate workflow. PRs that regress AUROC >0.02 are flagged | CI passes, benchmark gate enforced |
 | 6 | No script to regenerate all derived outputs (metrics_snapshot, evidence certs, etc.) deterministically | `make regenerate-all` target that runs full pipeline, benchmarks, and evidence generation from versioned inputs | Deterministic: `git diff --stat` shows zero unexpected changes |
 | 7 | Some `__init__.py` files in subpackages (benchmark, scoring, selection) may be empty or incomplete | Audit and populate all subpackage `__init__.py` exports; add `__all__` everywhere | `from openamp_foundry.benchmark import ...` works cleanly |
 | 8 | No contributor onboarding guide that specifically calls out safety review and claim policy | `CONTRIBUTING.md` updated with safety-first contribution checklist; PR template added | New contributor can open a safe first PR in <30 min |
@@ -201,4 +201,4 @@ Phase 4: Not started
 - `make ci` passes with benchmark gate
 - A new agent can read README → run demo → understand calibration flow → contribute safely in one session
 
-**Next loop:** Loop 5 — CI benchmark gate workflow (protect against scoring regressions).
+**Next loop:** Loop 6 — deterministic regenerate-all script.

scripts/benchmark_gate.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Benchmark regression gate for CI — fail CI if AUROC drops significantly."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def run_benchmark_gate(
+    baseline: dict,
+    current: dict,
+    tolerance: float = 0.02,
+    *,
+    report_file: str | Path | None = None,
+) -> int:
+    keys = ["standard", "phase3"]
+    lines: list[str] = []
+    lines.append("=== Benchmark Regression Gate ===")
+    lines.append(f"Tolerance: AUROC drop <= {tolerance}")
+    lines.append("")
+
+    worst_delta = 0.0
+    for key in keys:
+        b = baseline.get(key, {}).get("auroc", 0.0)
+        c = current.get(key, {}).get("auroc", 0.0)
+        d = b - c
+        flag = "FAIL" if d > tolerance else "PASS"
+        lines.append(
+            f"  [{flag:4s}] {key + '.auroc':30s} {b:.4f} → {c:.4f}  (Δ={d:+.4f})"
+        )
+        if d > worst_delta:
+            worst_delta = d
+
+    lines.append("")
+    if worst_delta > tolerance:
+        lines.append(
+            f"FAIL: AUROC regression detected — max Δ={worst_delta:.4f}"
+            f" exceeds tolerance {tolerance}"
+        )
+        lines.append("Investigate scoring changes before merging.")
+        report = "\n".join(lines)
+        print(report)
+        if report_file:
+            Path(report_file).write_text(report, encoding="utf-8")
+        return 1
+
+    lines.append(
+        f"PASS: All AUROC metrics within tolerance "
+        f"(max Δ={worst_delta:.4f}, limit={tolerance})"
+    )
+    report = "\n".join(lines)
+    print(report)
+    if report_file:
+        Path(report_file).write_text(report, encoding="utf-8")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Benchmark regression gate — checks AUROC hasn't dropped significantly."
+    )
+    parser.add_argument(
+        "--baseline",
+        default="outputs/metrics_snapshot.json",
+        help="Path to baseline metrics_snapshot.json",
+    )
+    parser.add_argument(
+        "--tolerance",
+        type=float,
+        default=0.02,
+        help="Maximum allowed AUROC drop (default: 0.02)",
+    )
+    parser.add_argument(
+        "--out",
+        default=None,
+        help="Optional path to write report file",
+    )
+    parser.add_argument(
+        "--n-bootstrap",
+        type=int,
+        default=200,
+        help="Bootstrap iterations for benchmarks (default: 200, CI-friendly)",
+    )
+    args = parser.parse_args(argv)
+
+    baseline_path = Path(args.baseline)
+    if not baseline_path.exists():
+        print(f"ERROR: Baseline not found at {baseline_path}", file=sys.stderr)
+        print("Run 'make metrics-snapshot' first to generate baseline.", file=sys.stderr)
+        return 2
+
+    baseline = json.loads(baseline_path.read_text(encoding="utf-8"))
+
+    from openamp_foundry.benchmark.metrics_snapshot import build_metrics_snapshot
+
+    print("Running benchmarks (all 6)...")
+    current = build_metrics_snapshot(n_bootstrap=args.n_bootstrap)
+    print("Done.\n")
+
+    return run_benchmark_gate(
+        baseline, current, tolerance=args.tolerance, report_file=args.out
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_benchmark_gate.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import unittest.mock
+from pathlib import Path
+
+from scripts.benchmark_gate import run_benchmark_gate
+
+
+def _snapshot(auroc: float) -> dict:
+    return {
+        "standard": {"auroc": auroc, "auprc": 0.8, "n_positives": 95, "n_negatives": 96},
+        "phase3": {"auroc": auroc - 0.0384, "auprc": 0.76, "n_positives": 95, "n_negatives": 96},
+    }
+
+
+class TestRunBenchmarkGate:
+    def test_passes_when_auroc_unchanged(self):
+        snap = _snapshot(0.7832)
+        assert run_benchmark_gate(snap, snap) == 0
+
+    def test_passes_when_auroc_improves(self):
+        baseline = _snapshot(0.7832)
+        current = _snapshot(0.7900)
+        assert run_benchmark_gate(baseline, current) == 0
+
+    def test_passes_within_tolerance(self):
+        baseline = _snapshot(0.7832)
+        current = _snapshot(0.7700)
+        assert run_benchmark_gate(baseline, current, tolerance=0.02) == 0
+
+    def test_fails_when_auroc_drops_below_tolerance(self):
+        baseline = _snapshot(0.7832)
+        current = _snapshot(0.7550)
+        assert run_benchmark_gate(baseline, current, tolerance=0.02) == 1
+
+    def test_fails_when_both_metrics_regress(self):
+        baseline = _snapshot(0.7832)
+        current = _snapshot(0.7000)
+        assert run_benchmark_gate(baseline, current, tolerance=0.02) == 1
+
+    def test_fails_when_tolerance_is_zero_and_auroc_drops(self):
+        baseline = _snapshot(0.7832)
+        current = _snapshot(0.7831)
+        assert run_benchmark_gate(baseline, current, tolerance=0.0) == 1
+
+    def test_fails_when_baseline_missing_metrics(self):
+        assert run_benchmark_gate({}, _snapshot(0.7832)) == 0
+
+    def test_writes_report_file(self, tmp_path):
+        report = tmp_path / "report.md"
+        snap = _snapshot(0.7832)
+        rc = run_benchmark_gate(snap, snap, report_file=report)
+        assert rc == 0
+        assert report.exists()
+        content = report.read_text(encoding="utf-8")
+        assert "PASS" in content
+
+    def test_writes_report_file_on_failure(self, tmp_path):
+        report = tmp_path / "report.md"
+        baseline = _snapshot(0.7832)
+        current = _snapshot(0.7000)
+        rc = run_benchmark_gate(baseline, current, report_file=report)
+        assert rc == 1
+        content = report.read_text(encoding="utf-8")
+        assert "FAIL" in content
+
+
+class TestBenchmarkGateCLI:
+    def test_missing_baseline_returns_2(self, tmp_path):
+        fake = tmp_path / "nonexistent.json"
+        result = subprocess.run(
+            [sys.executable, "scripts/benchmark_gate.py", "--baseline", str(fake)],
+            capture_output=True, text=True,
+            cwd=Path(__file__).resolve().parent.parent,
+        )
+        assert result.returncode == 2
+
+    def test_passes_with_committed_baseline(self):
+        result = subprocess.run(
+            [sys.executable, "scripts/benchmark_gate.py",
+             "--baseline", "outputs/metrics_snapshot.json",
+             "--n-bootstrap", "50"],
+            capture_output=True, text=True,
+            cwd=Path(__file__).resolve().parent.parent,
+        )
+        print(result.stdout)
+        if result.returncode != 0:
+            print(result.stderr)
+        assert result.returncode == 0
+        assert "PASS" in result.stdout
+
+    def test_fails_with_artificially_low_tolerance(self, tmp_path):
+        baseline = _snapshot(0.7832)
+        current = _snapshot(0.7810)
+        low = tmp_path / "baseline.json"
+        low.write_text(json.dumps(baseline), encoding="utf-8")
+
+        with unittest.mock.patch(
+            "openamp_foundry.benchmark.metrics_snapshot.build_metrics_snapshot",
+            return_value=current,
+        ):
+            from scripts.benchmark_gate import main
+            rc = main(["--baseline", str(low), "--tolerance", "0.0"])
+            assert rc == 1
+
+    def test_out_flag_writes_report(self, tmp_path):
+        snap = _snapshot(0.7832)
+        baseline_file = tmp_path / "baseline.json"
+        baseline_file.write_text(json.dumps(snap), encoding="utf-8")
+
+        with unittest.mock.patch(
+            "openamp_foundry.benchmark.metrics_snapshot.build_metrics_snapshot",
+            return_value=snap,
+        ):
+            report = tmp_path / "gate.md"
+            from scripts.benchmark_gate import main
+            rc = main(["--baseline", str(baseline_file), "--out", str(report)])
+            assert rc == 0
+            assert report.exists()
+            assert "PASS" in report.read_text(encoding="utf-8")