diff --git a/Makefile b/Makefile index 9eaced33..b170e83b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-baseline bench-hidden-active generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter +.PHONY: help demo test lint ci clean bench-leakage bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -28,6 +28,8 @@ help: @echo " make validate-scoring-strict AUROC with scrambled-decoy strict benchmark" @echo " make bench-leakage Check for near-duplicates between candidates and refs" @echo " make bench-baseline Hidden-positive recovery benchmark (demo set)" + @echo " make bench-cluster-split Cluster-aware bootstrap CI (de-inflates near-duplicates)" + @echo " make bench-expert-ablation Expert composite vs ensemble ablation (honesty check)" @echo " make bench-hidden-active Hidden-positive recovery on mixed benchmark set" @echo "" @echo " make wave0-5-gate-check Run Wave 0.5 gates W0.5-1 through W0.5-7" @@ -134,6 +136,18 @@ validate-scoring-phase3: --benchmark-type standard \ --out outputs/validate_scoring_report_phase3.json +bench-cluster-split: + PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench cluster-split + --amp-csv examples/validation/known_amps.csv + --decoy-csv examples/validation/random_background.csv + --out outputs/cluster_split_report.json + +bench-expert-ablation: + PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench expert-ablation + --amp-csv examples/validation/known_amps.csv + --decoy-csv examples/validation/random_background.csv + --out outputs/expert_ablation_report.json + validate-scoring-strict: PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli validate-scoring \ --amp-csv examples/validation/known_amps.csv \ diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 287e21c4..594c2a44 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -39,7 +39,7 @@ The purpose of the added layers is not simulation theater. It is to improve whic | `openamp_foundry.scoring` | activity, safety, novelty, synthesis, ensemble scoring | | `openamp_foundry.selection` | ranking and diversity selection | | `openamp_foundry.evidence` | JSON certificate generation and validation | -| `openamp_foundry.benchmark` | leakage checks, cluster-split benchmark, and evaluation scaffolding | +| `openamp_foundry.benchmark` | leakage checks, cluster-split benchmark, expert ablation benchmark, and evaluation scaffolding | | `openamp_foundry.generators` | safe, bounded toy candidate generation | | `openamp_foundry.simulation` | membrane/selectivity/stability proxy modeling (scaffolded) | diff --git a/docs/BENCHMARK_CARD.md b/docs/BENCHMARK_CARD.md index 871b844a..fc486497 100644 --- a/docs/BENCHMARK_CARD.md +++ b/docs/BENCHMARK_CARD.md @@ -72,6 +72,43 @@ indolicidin/analog/lys-analog, BMAP-27/PMAP-36/bmap-fragment, aurein-1/3, cecrop apidaecin-Ia/Ib, RsAFP-1/2, plectasin/eurocin, piscidin-1/3, buforin/buforin-II, dermaseptin-S1/S3-fragment, KWK-template/WKL. + +## Expert Ablation (added 2026-07-01) + +> Run: `make bench-expert-ablation` + +The expert composite scorer (`scoring/expert.py`, 7 components) was benchmarked +against the simple ensemble to test whether its additional complexity earns its keep. + +| Scorer | AUROC | CI₉₅ | Delta vs ensemble | +|--------|:-----:|:----:|:-----------------:| +| **Ensemble** (pipeline.yaml) | **0.7832** | 0.717–0.8423 | — | +| Expert composite | 0.7360 | 0.6604–0.8037 | **−0.0472** | +| **Ensemble** (phase3.yaml) | **0.7448** | 0.6741–0.8118 | — | +| Expert composite | 0.7360 | 0.6604–0.8037 | −0.0088 | + +**Per-component AUROC:** + +| Component | AUROC | Signal class | +|-----------|:-----:|:------------:| +| activity | 0.8137 | Signal-bearing | +| selectivity_proxy | 0.7729 | Signal-bearing | +| hinge_selectivity | 0.5180 | Near-zero | +| boman_activity | 0.4620 | Near-zero | +| synthesis | 0.4228 | Anti-signal | +| safety | 0.3487 | Anti-signal | +| serum_stability | 0.2231 | Anti-signal | + +**Verdict:** Expert composite does NOT improve AMP-vs-decoy discrimination. Three +components (safety, serum_stability, synthesis) are anti-signal: known AMPs score +worse than random decoys on these axes because real AMPs have extreme biophysical +properties (high charge, high hydrophobic moment, many protease sites) that these +scorers penalise. The ensemble remains the primary synthesis gate. + +**Caveat:** This tests binary AMP/non-AMP discrimination only. The expert components +may add value for within-AMP ranking (selective vs hemolytic) — a within-AMP benchmark +is the appropriate next test. + ## Known Biases | Bias | Impact | Mitigation | diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index 31a2e496..bfd85810 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -163,11 +163,77 @@ Machine-readable: `outputs/wave0_5_external_predict_results.csv`, `outputs/wave0 --- +--- + +## Expert Ablation Benchmark (v0.5.x — added 2026-07-01) + +> The expert composite scorer (`scoring/expert.py`) adds four components beyond the +> simple ensemble: selectivity, serum stability, helix-hinge, and k-mer motif novelty. +> This ablation tests whether those additions improve binary AMP-vs-decoy discrimination +> or are complexity that does not earn its keep. +> +> Run: `make bench-expert-ablation` + +| Metric | Pipeline (pipeline.yaml) | Phase3 (phase3.yaml) | +|--------|:-----------------------:|:-------------------:| +| Ensemble AUROC | 0.7832 | 0.7448 | +| Ensemble CI₉₅ | 0.717–0.8423 | 0.6741–0.8118 | +| Expert composite AUROC | 0.7360 | 0.7360 | +| Expert CI₉₅ | 0.6604–0.8037 | 0.6604–0.8037 | +| **Delta (expert − ensemble)** | **−0.0472** | **−0.0088** | +| Verdict | Expert LOWER | Within ±0.02 | + +**Per-component AUROC** (pipeline config, n=191): + +| Component | AUROC | Above random | Classification | +|-----------|:-----:|:------------:|:--------------:| +| activity | 0.8137 | +0.3137 | **Signal-bearing** | +| selectivity_proxy | 0.7729 | +0.2729 | **Signal-bearing** | +| hinge_selectivity | 0.5180 | +0.0180 | Near-zero | +| novelty | 0.5000 | 0.0000 | Near-zero | +| motif_novelty | 0.5000 | 0.0000 | Near-zero | +| boman_activity | 0.4620 | −0.0380 | Near-zero | +| synthesis | 0.4228 | −0.0772 | **Anti-signal** | +| safety | 0.3487 | −0.1513 | **Anti-signal** | +| serum_stability | 0.2231 | −0.2769 | **Anti-signal** | + +**Key finding:** The expert composite scores **lower** than the simple ensemble on +binary AMP-vs-decoy discrimination (delta = −0.0472). Three expert components are +anti-signal: `safety`, `serum_stability`, and `synthesis` score random decoys higher +than known AMPs. This is expected and not a bug: real AMPs have high hydrophobic +moment and charge (features the safety scorer penalises), many interior protease +sites (low serum stability), and often contain repeat runs or aggregation-prone +segments (low synthesis feasibility). Random decoys drawn from Swiss-Prot +frequencies tend to have more moderate biophysical properties. + +**What this means for the pipeline:** + +1. The expert composite should NOT replace the ensemble for AMP/non-AMP triage. +2. The ensemble (activity + safety + synthesis + novelty + Boman) remains the + primary synthesis gate, as it has higher discriminative power. +3. The expert components may still add value for **within-AMP ranking** (selectivity + and safety differentiation among candidates that already pass the activity gate) + — but this has not been demonstrated and should not be assumed. +4. The `boman_activity` scorer (AUROC 0.462, below random) does not discriminate + AMPs from random decoys on the expanded benchmark. It contributes to the ensemble + only through the disagreement signal, not through independent discrimination. +5. `motif_novelty` and `novelty` are 0.5 by construction (no k-mer index, no + references in this benchmark) — they are correctly neutral, not noise. + +**Honest limitation:** This benchmark measures binary AMP-vs-decoy discrimination +only. The expert composite's selectivity, safety, and synthesis components are +designed for within-AMP candidate differentiation, not for separating AMPs from +non-AMPs. A within-AMP ranking benchmark (comparing selective vs hemolytic AMPs) +would be a more appropriate test for those components and is a candidate for the +next loop. + +--- + ## Test Suite | Metric | Value | |--------|-------| -| Total tests | 1343 | +| Total tests | 1435 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | @@ -195,4 +261,5 @@ Machine-readable: `outputs/wave0_5_external_predict_results.csv`, `outputs/wave0 | 2026-06-29 | Wave 0.5b: 23-candidate safety-optimized shortlist (SEED-020–024, no aromatics) | OpenAMP Wave 0.5b | | 2026-06-29 | External predictor results filled from wave05_combined_consensus.csv; all 7 gates PASS | OpenAMP Wave 0.5 | | 2026-06-29 | Wave 0.5 scaffold diversification — 24-candidate Wave 1 panel across 14 families | OpenAMP Wave 0.5 | +| 2026-07-01 | Expert ablation benchmark added: expert composite AUROC 0.736 vs ensemble 0.7832 (delta −0.0472); 3 components anti-signal; ensemble remains primary gate | OpenAMP loop | | 2026-06-29 | Initial — expanded benchmark (PR #110) | OpenAMP CI | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 4229adc1..77be9e73 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -136,6 +136,16 @@ Implemented during the pre-wet-lab improvement loop (PRs #31–#54): - Wave 0.5 gates W0.5-1 through W0.5-7 implemented (`src/openamp_foundry/gates/wave0_5_gate_checker.py`) - Discovery probability impact: from 7 correlated families → 14 independent families; reduces correlated-failure risk for wet-lab batch +## v0.5.8 — Expert Ablation Benchmark ✓ (2026-07-01) + +- `run_expert_ablation_benchmark()` in `benchmark/retrospective.py` — scores all 95 AMPs + 96 decoys with both the simple ensemble and the 7-component expert composite +- `bench expert-ablation` CLI command + `make bench-expert-ablation` target +- Per-component AUROC attribution: identifies signal-bearing (activity 0.814, selectivity 0.773), near-zero (hinge, novelty, boman, motif), and anti-signal (safety 0.349, serum_stability 0.223, synthesis 0.423) components +- **Key finding:** Expert composite AUROC 0.736 < ensemble AUROC 0.783 (delta −0.047). The added complexity does NOT improve AMP-vs-decoy discrimination. Three components are anti-signal because real AMPs have extreme biophysical properties that safety/stability/synthesis scorers penalise. +- Ensemble remains primary synthesis gate. Expert composite may add value for within-AMP ranking (not yet demonstrated). +- `make bench-cluster-split` Makefile target added (was missing despite CLI command existing) +- 14 new tests in `test_expert_ablation.py`; 1435 total tests + ## v1.0 — Validated dry-lab-to-wet-lab loop - independently reviewed assay batch (expert_review.yml GitHub issue template); diff --git a/src/openamp_foundry/benchmark/retrospective.py b/src/openamp_foundry/benchmark/retrospective.py index c6c87adc..ff3df73f 100644 --- a/src/openamp_foundry/benchmark/retrospective.py +++ b/src/openamp_foundry/benchmark/retrospective.py @@ -555,3 +555,204 @@ def run_cluster_split_benchmark( "Wet-lab validation remains mandatory." ), } + + +def run_expert_ablation_benchmark( + amp_csv: str | Path, + decoy_csv: str | Path, + config_path: str | Path = "configs/pipeline.yaml", + n_bootstrap: int = 2000, +) -> dict: + """Expert-vs-ensemble ablation: does the expert composite beat the simple ensemble? + + The expert composite (scoring/expert.py) adds four components beyond the simple + ensemble: selectivity, serum stability, helix-hinge, and k-mer motif novelty. + The simple ensemble uses only activity, safety, synthesis, novelty, and Boman. + + If the expert composite does not outperform the simple ensemble on AUROC, the + added complexity is not earning its keep — it may be noise dressed as sophistication. + + This benchmark also computes per-component AUROC (each component score alone vs + the label) to identify which axes carry discriminative signal and which do not. + + Per AGENTS.md §7 (Benchmarks must be adversarial) and §7.5 (No simulation theater): + every added modeling layer must justify itself against simpler baselines. + + Args: + amp_csv: CSV of known AMPs (id, sequence, ...). + decoy_csv: CSV of decoy peptides. + config_path: Pipeline config for the simple ensemble weights. + n_bootstrap: Bootstrap resample count for CIs. + + Returns: + Dict with AUROC for ensemble and expert composite, per-component AUROCs, + delta, bootstrap CIs, and an honest verdict. + """ + from openamp_foundry.config import load_config + from openamp_foundry.features.physchem import compute_features + from openamp_foundry.scoring.activity import activity_likeness_score + from openamp_foundry.scoring.boman import boman_activity_score, model_disagreement + from openamp_foundry.scoring.ensemble import ensemble_score + from openamp_foundry.scoring.expert import EXPERT_WEIGHTS, expert_score + from openamp_foundry.scoring.novelty import novelty_score + from openamp_foundry.scoring.safety import safety_score + from openamp_foundry.scoring.stability import serum_stability_score + from openamp_foundry.scoring.synthesis import synthesis_feasibility_score + + config = load_config(config_path) + weights = config["weights"] + + rows = [] + for path, label in [(amp_csv, 1), (decoy_csv, 0)]: + with open(path, newline="", encoding="utf-8") as f: + for row in csv.DictReader(f): + seq = row["sequence"].strip().upper() + features = compute_features(seq) + act = activity_likeness_score(features) + safe = safety_score(features) + valid = all(aa in "ACDEFGHIKLMNPQRSTVWY" for aa in seq) + synth = synthesis_feasibility_score(features, valid_sequence=valid) + nov, _ = novelty_score(seq, []) + boman_act = boman_activity_score(seq) + stability = serum_stability_score(features) + raw = { + "activity": act, "safety": safe, + "synthesis": synth, "novelty": nov, + "boman_activity": boman_act, + "disagreement": model_disagreement(act, boman_act), + "serum_stability": stability, + "selectivity_proxy": features.get("selectivity_proxy", 1.0), + } + ens = ensemble_score(raw, weights) + + # Expert composite — no k-mer index (benchmark measures composition-based + # signal only; motif novelty defaults to 1.0 for all sequences equally, + # which correctly removes it as a discriminator and tests the other 6 axes). + exp = expert_score(seq, features=features) + + rows.append({ + "id": row["id"], + "label": label, + "ensemble": ens, + "expert_composite": exp.composite, + # Per-component scores for signal attribution + "activity": act, + "safety": safe, + "synthesis": synth, + "novelty": nov, + "boman_activity": boman_act, + "serum_stability": stability, + "selectivity_proxy": features.get("selectivity_proxy", 1.0), + "hinge_selectivity": exp.components["hinge_selectivity"], + "motif_novelty": exp.components["motif_novelty"], + }) + + pos = [r for r in rows if r["label"] == 1] + neg = [r for r in rows if r["label"] == 0] + + ens_auroc = round(_auc_wilcoxon( + [r["ensemble"] for r in pos], [r["ensemble"] for r in neg] + ), 4) + exp_auroc = round(_auc_wilcoxon( + [r["expert_composite"] for r in pos], [r["expert_composite"] for r in neg] + ), 4) + + ens_ci = _bootstrap_auroc_ci( + [r["ensemble"] for r in pos], [r["ensemble"] for r in neg], + n_bootstrap=n_bootstrap, + ) + exp_ci = _bootstrap_auroc_ci( + [r["expert_composite"] for r in pos], [r["expert_composite"] for r in neg], + n_bootstrap=n_bootstrap, + ) + + delta = round(exp_auroc - ens_auroc, 4) + + # Per-component AUROC: which individual axes discriminate AMPs from decoys? + component_cols = [ + "activity", "safety", "synthesis", "novelty", + "boman_activity", "serum_stability", "selectivity_proxy", + "hinge_selectivity", "motif_novelty", + ] + per_component = {} + for col in component_cols: + pos_vals = [r[col] for r in pos] + neg_vals = [r[col] for r in neg] + auroc = round(_auc_wilcoxon(pos_vals, neg_vals), 4) + per_component[col] = { + "auroc": auroc, + "above_random": round(auroc - 0.5, 4), + } + + # Honest verdict: does the expert composite earn its complexity? + if delta > 0.02 and exp_ci["ci_lo"] > ens_ci["ci_lo"]: + verdict = ( + f"Expert composite outperforms ensemble by {delta:+.4f} AUROC. " + "Added complexity is justified by improved discrimination." + ) + elif abs(delta) <= 0.02: + verdict = ( + f"Expert composite ({exp_auroc:.4f}) and ensemble ({ens_auroc:.4f}) " + f"are within ±0.02 AUROC. The expert composite's additional components " + "(selectivity, serum stability, helix hinge, motif novelty) do NOT " + "materially improve AMP-vs-decoy discrimination on this benchmark. " + "They may still add value for candidate selection within the AMP-like " + "envelope (e.g. safety ranking), but they are not better at the binary " + "AMP/non-AMP task. The simple ensemble remains the primary synthesis gate." + ) + else: + verdict = ( + f"Expert composite ({exp_auroc:.4f}) scores LOWER than ensemble " + f"({ens_auroc:.4f}) by {delta:+.4f} AUROC. The added components " + "degrade discrimination on this benchmark. The expert composite should " + "NOT replace the ensemble for AMP/non-AMP triage without further evidence." + ) + + # Component signal summary: which axes carry the discrimination? + sorted_components = sorted( + per_component.items(), key=lambda x: x[1]["above_random"], reverse=True + ) + signal_bearing = [ + name for name, info in sorted_components if info["above_random"] > 0.05 + ] + noise_components = [ + name for name, info in sorted_components if abs(info["above_random"]) <= 0.05 + ] + anti_signal = [ + name for name, info in sorted_components if info["above_random"] < -0.05 + ] + + return { + "benchmark": "expert_ablation", + "n_positives": len(pos), + "n_negatives": len(neg), + "n_total": len(rows), + "config_path": str(config_path), + "expert_weights": EXPERT_WEIGHTS, + "ensemble_auroc": ens_auroc, + "ensemble_ci95_lo": ens_ci["ci_lo"], + "ensemble_ci95_hi": ens_ci["ci_hi"], + "expert_auroc": exp_auroc, + "expert_ci95_lo": exp_ci["ci_lo"], + "expert_ci95_hi": exp_ci["ci_hi"], + "delta_auroc": delta, + "per_component_auroc": per_component, + "signal_bearing_components": signal_bearing, + "near_zero_components": noise_components, + "anti_signal_components": anti_signal, + "verdict": verdict, + "design_note": ( + "The expert composite (scoring/expert.py) adds selectivity, serum stability, " + "hinge selectivity, and k-mer motif novelty on top of the ensemble axes. " + "This ablation measures whether those additions improve binary AMP-vs-decoy " + "discrimination. No k-mer index is used (motif_novelty = 1.0 for all), " + "isolating the composition-based components. AUROC > 0.70 does NOT imply " + "nominated candidates are antimicrobial." + ), + "disclaimer": ( + "Ablation results are computational only. A negative result does not mean " + "the expert components are useless — they may improve within-AMP ranking " + "(safety, selectivity) even if they do not improve AMP-vs-decoy separation. " + "Wet-lab validation remains mandatory." + ), + } diff --git a/src/openamp_foundry/cli/commands/benchmark.py b/src/openamp_foundry/cli/commands/benchmark.py index 84556bdc..85a0efbf 100644 --- a/src/openamp_foundry/cli/commands/benchmark.py +++ b/src/openamp_foundry/cli/commands/benchmark.py @@ -117,3 +117,37 @@ def _run_cluster_split_bench(args: argparse.Namespace) -> int: } print(_json.dumps(summary, indent=2)) return 0 + + +def _run_expert_ablation_bench(args: argparse.Namespace) -> int: + """Run expert-vs-ensemble ablation benchmark.""" + import json as _json + from openamp_foundry.benchmark.retrospective import run_expert_ablation_benchmark + from openamp_foundry.utils.io import write_json + + result = run_expert_ablation_benchmark( + amp_csv=args.amp_csv, + decoy_csv=args.decoy_csv, + config_path=args.config, + n_bootstrap=args.n_bootstrap, + ) + if args.out: + write_json(args.out, result) + summary = { + "status": "ok", + "benchmark": "expert_ablation", + "n_positives": result["n_positives"], + "n_negatives": result["n_negatives"], + "ensemble_auroc": result["ensemble_auroc"], + "ensemble_ci95": f"{result['ensemble_ci95_lo']}-{result['ensemble_ci95_hi']}", + "expert_auroc": result["expert_auroc"], + "expert_ci95": f"{result['expert_ci95_lo']}-{result['expert_ci95_hi']}", + "delta_auroc": result["delta_auroc"], + "signal_bearing_components": result["signal_bearing_components"], + "near_zero_components": result["near_zero_components"], + "anti_signal_components": result["anti_signal_components"], + "verdict": result["verdict"], + "out": args.out, + } + print(_json.dumps(summary, indent=2)) + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index bc4f9d63..79c4f0e1 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -1,7 +1,7 @@ from __future__ import annotations from openamp_foundry.cli.commands.core import _run_generate_batch -from openamp_foundry.cli.commands.benchmark import _run_bench, _run_validate_scoring, _run_cluster_split_bench +from openamp_foundry.cli.commands.benchmark import _run_bench, _run_validate_scoring, _run_cluster_split_bench, _run_expert_ablation_bench from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc @@ -105,6 +105,41 @@ def build_parser() -> argparse.ArgumentParser: help="Optional JSON output path.", ) + expert_ablation = bench_sub.add_parser( + "expert-ablation", + help=( + "Expert-vs-ensemble ablation: does the 7-component expert composite " + "(selectivity, serum stability, helix hinge, motif novelty) beat the " + "simple ensemble on AMP-vs-decoy discrimination? Reports per-component " + "AUROC to identify signal-bearing vs noise components." + ), + ) + expert_ablation.add_argument( + "--amp-csv", + default="examples/validation/known_amps.csv", + help="CSV of known AMPs (id, sequence, ...).", + ) + expert_ablation.add_argument( + "--decoy-csv", + default="examples/validation/random_background.csv", + help="CSV of decoy peptides.", + ) + expert_ablation.add_argument( + "--config", + default="configs/pipeline.yaml", + ) + expert_ablation.add_argument( + "--n-bootstrap", + type=int, + default=2000, + help="Bootstrap resample count (default: 2000).", + ) + expert_ablation.add_argument( + "--out", + required=False, + help="Optional JSON output path.", + ) + generate = sub.add_parser( "generate-batch", help=( @@ -550,6 +585,8 @@ def main(argv: list[str] | None = None) -> int: if args.command == "bench": if args.bench_command == "cluster-split": return _run_cluster_split_bench(args) + if args.bench_command == "expert-ablation": + return _run_expert_ablation_bench(args) return _run_bench(args) if args.command == "generate-batch": diff --git a/tests/test_expert_ablation.py b/tests/test_expert_ablation.py new file mode 100644 index 00000000..2e38891a --- /dev/null +++ b/tests/test_expert_ablation.py @@ -0,0 +1,134 @@ +"""Expert-vs-ensemble ablation benchmark tests — Phase 2 honesty requirement. + +AGENTS.md §7.5: "No simulation theater — every proxy module must justify itself +against cheap heuristic baselines before it affects selection." + +These tests verify that: +1. The ablation benchmark runs on the real validation set. +2. Both ensemble and expert AUROCs are in [0, 1]. +3. Per-component AUROCs are reported for all 9 axes. +4. The delta and verdict are internally consistent. +5. The verdict does not overclaim when the expert composite underperforms. +6. The benchmark handles edge cases (empty sets, single-class). + +All results are computational. No biological activity is implied. +""" +from __future__ import annotations + +import csv +import pytest + +from openamp_foundry.benchmark.retrospective import run_expert_ablation_benchmark + + +AMP_CSV = "examples/validation/known_amps.csv" +DECOY_CSV = "examples/validation/random_background.csv" + +EXPECTED_COMPONENTS = [ + "activity", "safety", "synthesis", "novelty", + "boman_activity", "serum_stability", "selectivity_proxy", + "hinge_selectivity", "motif_novelty", +] + + +class TestExpertAblationBenchmark: + @pytest.fixture(scope="class") + def result(self): + return run_expert_ablation_benchmark( + AMP_CSV, DECOY_CSV, n_bootstrap=200, + ) + + def test_benchmark_runs_on_real_data(self, result): + assert result["benchmark"] == "expert_ablation" + assert result["n_positives"] == 95 + assert result["n_negatives"] == 96 + assert result["n_total"] == 191 + + def test_ensemble_auroc_in_valid_range(self, result): + assert 0.0 <= result["ensemble_auroc"] <= 1.0 + # The ensemble should have signal (above 0.70 gate) + assert result["ensemble_auroc"] >= 0.70 + + def test_expert_auroc_in_valid_range(self, result): + assert 0.0 <= result["expert_auroc"] <= 1.0 + + def test_delta_is_consistent(self, result): + delta = result["delta_auroc"] + assert delta == pytest.approx( + result["expert_auroc"] - result["ensemble_auroc"], abs=1e-4 + ) + + def test_per_component_auroc_reported(self, result): + per_comp = result["per_component_auroc"] + for name in EXPECTED_COMPONENTS: + assert name in per_comp + assert 0.0 <= per_comp[name]["auroc"] <= 1.0 + + def test_component_classification_is_consistent(self, result): + """Signal-bearing, near-zero, and anti-signal partition should cover all 9.""" + all_classified = set( + result["signal_bearing_components"] + + result["near_zero_components"] + + result["anti_signal_components"] + ) + assert all_classified == set(EXPECTED_COMPONENTS) + + def test_verdict_matches_delta_direction(self, result): + """The verdict text must be consistent with the delta sign.""" + verdict = result["verdict"] + if result["delta_auroc"] < -0.02: + assert "LOWER" in verdict or "degrade" in verdict + elif abs(result["delta_auroc"]) <= 0.02: + assert "within" in verdict or "NOT" in verdict + else: + assert "outperforms" in verdict + + def test_ci_bounds_are_ordered(self, result): + assert result["ensemble_ci95_lo"] <= result["ensemble_ci95_hi"] + assert result["expert_ci95_lo"] <= result["expert_ci95_hi"] + + def test_disclaimer_present(self, result): + assert "wet-lab" in result["disclaimer"].lower() + assert "computational" in result["disclaimer"].lower() + + def test_design_note_present(self, result): + assert "k-mer index" in result["design_note"] or "motif" in result["design_note"] + + def test_expert_weights_reported(self, result): + """Expert weights should be in the output for auditability.""" + assert "activity_consensus" in result["expert_weights"] + assert "selectivity" in result["expert_weights"] + assert "hinge_selectivity" in result["expert_weights"] + assert "motif_novelty" in result["expert_weights"] + + +class TestExpertAblationEdgeCases: + def test_empty_amp_set_returns_half_auroc(self, tmp_path): + amp = tmp_path / "empty.csv" + amp.write_text("id,sequence\n") + decoy = tmp_path / "decoy.csv" + decoy.write_text("id,sequence\nD1,KWKLFKKIGAVLKVL\n") + result = run_expert_ablation_benchmark(amp, decoy, n_bootstrap=10) + # No positives → AUROC = 0.5 by convention + assert result["ensemble_auroc"] == 0.5 + assert result["expert_auroc"] == 0.5 + + def test_single_class_returns_half(self, tmp_path): + """If all sequences are positives (no decoys), AUROC = 0.5.""" + amp = tmp_path / "amp.csv" + amp.write_text("id,sequence\nA1,KWKLFKKIGAVLKVL\nA2,GIGKFLHSAKKFGKAFVGEIMNS\n") + decoy = tmp_path / "empty_decoy.csv" + decoy.write_text("id,sequence\n") + result = run_expert_ablation_benchmark(amp, decoy, n_bootstrap=10) + assert result["ensemble_auroc"] == 0.5 + assert result["expert_auroc"] == 0.5 + + def test_perfect_separation(self, tmp_path): + """Two clearly separable sequences should give high AUROC for both scorers.""" + amp = tmp_path / "amp.csv" + amp.write_text("id,sequence\nA1,KWKLFKKIGAVLKVL\n") + decoy = tmp_path / "decoy.csv" + decoy.write_text("id,sequence\nD1,DEDEDEDEDEDEDE\n") + result = run_expert_ablation_benchmark(amp, decoy, n_bootstrap=10) + assert result["ensemble_auroc"] >= 0.9 + assert result["expert_auroc"] >= 0.9