From 59ce24f9e914e9ad5a7f152be5269251498d62dc Mon Sep 17 00:00:00 2001 From: j Date: Sun, 5 Jul 2026 00:37:27 +0700 Subject: [PATCH] =?UTF-8?q?v0.5.24=20=E2=80=94=20Benchmark=20regression=20?= =?UTF-8?q?gate=20for=20CI=20(Loop=205)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add CI benchmark gate that fails PRs when AUROC drops > 0.02 vs committed baseline. New: - scripts/benchmark_gate.py: standalone script that runs all 6 benchmarks via build_metrics_snapshot(), compares standard/phase3 AUROC against committed baseline, exits 0 (PASS) / 1 (FAIL) / 2 (no baseline) - tests/test_benchmark_gate.py: 13 tests covering pass/fail/report/CLI paths - Makefile bench-gate target: make bench-gate runs the gate with tolerance 0.02 CI changes (.github/workflows/ci.yml): - New step 'Benchmark regression gate' between leakage and hidden-active checks - Runs 'make bench-gate' — fails CI if standard.auroc drops > 0.02 Total tests: 1682 passing (+13 from v0.5.23) docs/50_LOOP_PLAN.md: Loop 5 marked complete, position advanced to Loop 6 --- .github/workflows/ci.yml | 3 + Makefile | 9 ++- docs/50_LOOP_PLAN.md | 4 +- scripts/benchmark_gate.py | 109 +++++++++++++++++++++++++++++++ tests/test_benchmark_gate.py | 123 +++++++++++++++++++++++++++++++++++ 5 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 scripts/benchmark_gate.py create mode 100644 tests/test_benchmark_gate.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79fd6d44..61b98cd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,9 @@ jobs: - name: Leakage check (informational) run: make bench-leakage + - name: Benchmark regression gate (AUROC drop <= 0.02 vs committed baseline) + run: make bench-gate + - name: Hidden-active benchmark (require EF >= 1.5 at k=5) run: | PYTHONPATH=src python -c " diff --git a/Makefile b/Makefile index 8e12e533..070ae382 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-selectivity bench-feature-decomp generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter +.PHONY: help demo test lint ci clean bench-leakage bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-selectivity bench-feature-decomp bench-gate generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -32,6 +32,7 @@ help: @echo " make bench-expert-ablation Expert composite vs ensemble ablation (honesty check)" @echo " make bench-selectivity Within-AMP selectivity (hemolytic vs selective)" @echo " make bench-feature-decomp Per-feature selective_vs_hemolytic decomposition" + @echo " make bench-gate Benchmark regression gate (AUROC drift check)" @echo " make bench-hidden-active Hidden-positive recovery on mixed benchmark set" @echo "" @echo " make wave0-5-gate-check Run Wave 0.5 gates W0.5-1 through W0.5-7" @@ -164,6 +165,12 @@ bench-triage: PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench triage \ --out outputs/triage_benchmark_report.json +bench-gate: + PYTHONPATH=src $(PYTHON) scripts/benchmark_gate.py \ + --baseline outputs/metrics_snapshot.json \ + --tolerance 0.02 \ + --out outputs/bench_gate_report.md + bench-feature-decomp: PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench feature-decomp \ --out outputs/feature_decomp_report.json diff --git a/docs/50_LOOP_PLAN.md b/docs/50_LOOP_PLAN.md index a3de8956..4b88ca68 100644 --- a/docs/50_LOOP_PLAN.md +++ b/docs/50_LOOP_PLAN.md @@ -186,12 +186,12 @@ Phase 4: Not started | 2 ✅ | README doesn't document calibration CLI or policy files; no 50-loop strategic plan exists | Updated README with calibration flow and repo map; 50-LOOP_PLAN.md created | 1652 pass, README reviewed | | 3 ✅ | Doc audit — verify freshness, deduplicate, archive stale content | Performed as part of Loop 1/2 structure work; calibration docs up to date | Docs consistent | | 4 ✅ | No end-to-end smoke test for the full calibration chain (panel→intake→gate→verdict) | `tests/test_calibration_e2e.py`: 14 new tests exercising Python API, CLI, and Makefile targets for both passing and failing paths | 1669 pass, full calibration flow verified | +| 5 ✅ | No CI benchmark regression gate — AUROC can silently degrade | `scripts/benchmark_gate.py`, `tests/test_benchmark_gate.py` (13 tests), `make bench-gate` target, CI step that fails on AUROC drop >0.02 | 1682 pass, `make bench-gate` exits 0, CI gate committed | ### Remaining (Phase 0) | Loop | Bottleneck | Deliverable | Verification | |------|-----------|-------------|-------------| -| 5 | No CI job that runs `make demo` + `make validate-scoring` + `make bench-triage` on every PR | `.github/workflows/ci.yml` expanded with benchmark gate workflow. PRs that regress AUROC >0.02 are flagged | CI passes, benchmark gate enforced | | 6 | No script to regenerate all derived outputs (metrics_snapshot, evidence certs, etc.) deterministically | `make regenerate-all` target that runs full pipeline, benchmarks, and evidence generation from versioned inputs | Deterministic: `git diff --stat` shows zero unexpected changes | | 7 | Some `__init__.py` files in subpackages (benchmark, scoring, selection) may be empty or incomplete | Audit and populate all subpackage `__init__.py` exports; add `__all__` everywhere | `from openamp_foundry.benchmark import ...` works cleanly | | 8 | No contributor onboarding guide that specifically calls out safety review and claim policy | `CONTRIBUTING.md` updated with safety-first contribution checklist; PR template added | New contributor can open a safe first PR in <30 min | @@ -201,4 +201,4 @@ Phase 4: Not started - `make ci` passes with benchmark gate - A new agent can read README → run demo → understand calibration flow → contribute safely in one session -**Next loop:** Loop 5 — CI benchmark gate workflow (protect against scoring regressions). +**Next loop:** Loop 6 — deterministic regenerate-all script. diff --git a/scripts/benchmark_gate.py b/scripts/benchmark_gate.py new file mode 100644 index 00000000..ddcbc5db --- /dev/null +++ b/scripts/benchmark_gate.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Benchmark regression gate for CI — fail CI if AUROC drops significantly.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def run_benchmark_gate( + baseline: dict, + current: dict, + tolerance: float = 0.02, + *, + report_file: str | Path | None = None, +) -> int: + keys = ["standard", "phase3"] + lines: list[str] = [] + lines.append("=== Benchmark Regression Gate ===") + lines.append(f"Tolerance: AUROC drop <= {tolerance}") + lines.append("") + + worst_delta = 0.0 + for key in keys: + b = baseline.get(key, {}).get("auroc", 0.0) + c = current.get(key, {}).get("auroc", 0.0) + d = b - c + flag = "FAIL" if d > tolerance else "PASS" + lines.append( + f" [{flag:4s}] {key + '.auroc':30s} {b:.4f} → {c:.4f} (Δ={d:+.4f})" + ) + if d > worst_delta: + worst_delta = d + + lines.append("") + if worst_delta > tolerance: + lines.append( + f"FAIL: AUROC regression detected — max Δ={worst_delta:.4f}" + f" exceeds tolerance {tolerance}" + ) + lines.append("Investigate scoring changes before merging.") + report = "\n".join(lines) + print(report) + if report_file: + Path(report_file).write_text(report, encoding="utf-8") + return 1 + + lines.append( + f"PASS: All AUROC metrics within tolerance " + f"(max Δ={worst_delta:.4f}, limit={tolerance})" + ) + report = "\n".join(lines) + print(report) + if report_file: + Path(report_file).write_text(report, encoding="utf-8") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Benchmark regression gate — checks AUROC hasn't dropped significantly." + ) + parser.add_argument( + "--baseline", + default="outputs/metrics_snapshot.json", + help="Path to baseline metrics_snapshot.json", + ) + parser.add_argument( + "--tolerance", + type=float, + default=0.02, + help="Maximum allowed AUROC drop (default: 0.02)", + ) + parser.add_argument( + "--out", + default=None, + help="Optional path to write report file", + ) + parser.add_argument( + "--n-bootstrap", + type=int, + default=200, + help="Bootstrap iterations for benchmarks (default: 200, CI-friendly)", + ) + args = parser.parse_args(argv) + + baseline_path = Path(args.baseline) + if not baseline_path.exists(): + print(f"ERROR: Baseline not found at {baseline_path}", file=sys.stderr) + print("Run 'make metrics-snapshot' first to generate baseline.", file=sys.stderr) + return 2 + + baseline = json.loads(baseline_path.read_text(encoding="utf-8")) + + from openamp_foundry.benchmark.metrics_snapshot import build_metrics_snapshot + + print("Running benchmarks (all 6)...") + current = build_metrics_snapshot(n_bootstrap=args.n_bootstrap) + print("Done.\n") + + return run_benchmark_gate( + baseline, current, tolerance=args.tolerance, report_file=args.out + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_benchmark_gate.py b/tests/test_benchmark_gate.py new file mode 100644 index 00000000..8693b924 --- /dev/null +++ b/tests/test_benchmark_gate.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import json +import subprocess +import sys +import unittest.mock +from pathlib import Path + +from scripts.benchmark_gate import run_benchmark_gate + + +def _snapshot(auroc: float) -> dict: + return { + "standard": {"auroc": auroc, "auprc": 0.8, "n_positives": 95, "n_negatives": 96}, + "phase3": {"auroc": auroc - 0.0384, "auprc": 0.76, "n_positives": 95, "n_negatives": 96}, + } + + +class TestRunBenchmarkGate: + def test_passes_when_auroc_unchanged(self): + snap = _snapshot(0.7832) + assert run_benchmark_gate(snap, snap) == 0 + + def test_passes_when_auroc_improves(self): + baseline = _snapshot(0.7832) + current = _snapshot(0.7900) + assert run_benchmark_gate(baseline, current) == 0 + + def test_passes_within_tolerance(self): + baseline = _snapshot(0.7832) + current = _snapshot(0.7700) + assert run_benchmark_gate(baseline, current, tolerance=0.02) == 0 + + def test_fails_when_auroc_drops_below_tolerance(self): + baseline = _snapshot(0.7832) + current = _snapshot(0.7550) + assert run_benchmark_gate(baseline, current, tolerance=0.02) == 1 + + def test_fails_when_both_metrics_regress(self): + baseline = _snapshot(0.7832) + current = _snapshot(0.7000) + assert run_benchmark_gate(baseline, current, tolerance=0.02) == 1 + + def test_fails_when_tolerance_is_zero_and_auroc_drops(self): + baseline = _snapshot(0.7832) + current = _snapshot(0.7831) + assert run_benchmark_gate(baseline, current, tolerance=0.0) == 1 + + def test_fails_when_baseline_missing_metrics(self): + assert run_benchmark_gate({}, _snapshot(0.7832)) == 0 + + def test_writes_report_file(self, tmp_path): + report = tmp_path / "report.md" + snap = _snapshot(0.7832) + rc = run_benchmark_gate(snap, snap, report_file=report) + assert rc == 0 + assert report.exists() + content = report.read_text(encoding="utf-8") + assert "PASS" in content + + def test_writes_report_file_on_failure(self, tmp_path): + report = tmp_path / "report.md" + baseline = _snapshot(0.7832) + current = _snapshot(0.7000) + rc = run_benchmark_gate(baseline, current, report_file=report) + assert rc == 1 + content = report.read_text(encoding="utf-8") + assert "FAIL" in content + + +class TestBenchmarkGateCLI: + def test_missing_baseline_returns_2(self, tmp_path): + fake = tmp_path / "nonexistent.json" + result = subprocess.run( + [sys.executable, "scripts/benchmark_gate.py", "--baseline", str(fake)], + capture_output=True, text=True, + cwd=Path(__file__).resolve().parent.parent, + ) + assert result.returncode == 2 + + def test_passes_with_committed_baseline(self): + result = subprocess.run( + [sys.executable, "scripts/benchmark_gate.py", + "--baseline", "outputs/metrics_snapshot.json", + "--n-bootstrap", "50"], + capture_output=True, text=True, + cwd=Path(__file__).resolve().parent.parent, + ) + print(result.stdout) + if result.returncode != 0: + print(result.stderr) + assert result.returncode == 0 + assert "PASS" in result.stdout + + def test_fails_with_artificially_low_tolerance(self, tmp_path): + baseline = _snapshot(0.7832) + current = _snapshot(0.7810) + low = tmp_path / "baseline.json" + low.write_text(json.dumps(baseline), encoding="utf-8") + + with unittest.mock.patch( + "openamp_foundry.benchmark.metrics_snapshot.build_metrics_snapshot", + return_value=current, + ): + from scripts.benchmark_gate import main + rc = main(["--baseline", str(low), "--tolerance", "0.0"]) + assert rc == 1 + + def test_out_flag_writes_report(self, tmp_path): + snap = _snapshot(0.7832) + baseline_file = tmp_path / "baseline.json" + baseline_file.write_text(json.dumps(snap), encoding="utf-8") + + with unittest.mock.patch( + "openamp_foundry.benchmark.metrics_snapshot.build_metrics_snapshot", + return_value=snap, + ): + report = tmp_path / "gate.md" + from scripts.benchmark_gate import main + rc = main(["--baseline", str(baseline_file), "--out", str(report)]) + assert rc == 0 + assert report.exists() + assert "PASS" in report.read_text(encoding="utf-8")