From 8e94f71c3d4cfafe3a727ec8e7fba2e15a4c4fce Mon Sep 17 00:00:00 2001 From: j Date: Wed, 1 Jul 2026 07:18:52 +0700 Subject: [PATCH] =?UTF-8?q?feat:=20selectivity=20benchmark=20=E2=80=94=20s?= =?UTF-8?q?afety=5Fscore=20AUROC=3D0.54=20(FAILED)=20vs=20hydrophobic=5Ffr?= =?UTF-8?q?action=20AUROC=3D0.81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the first selectivity benchmark to the OpenAMP Foundry pipeline. This directly addresses the v1.1 roadmap requirement: 'benchmark candidate triage against a reference panel that includes selective AMPs, hemolytic positives, inactive peptides, and random controls' and Phase 2.5 DoD: 'Membrane proxy benchmark — Distinguishes bacterial-selective vs clearly hemolytic reference peptides better than naive heuristics.' What was built: - examples/validation/selectivity_panel.csv: 10 selective + 8 hemolytic AMPs with literature hemolysis classifications (Zasloff, Habermann, Skerlavaj, etc.) - src/openamp_foundry/benchmark/selectivity.py: SelectivityResult dataclass + run_selectivity_benchmark() computing AUROC for safety_score, selectivity_proxy, hydrophobic_fraction, hydrophobic_moment, charge_density, and GRAVY - tests/test_selectivity_benchmark.py: 21 tests covering panel integrity, AUROC computation, blind spot detection, CLI integration - CLI: 'validate-selectivity' command + 'make validate-selectivity' target - Docs updated: METRICS_CURRENT, BENCHMARK_CARD, ROADMAP, ARCHITECTURE Key finding (honest, not flattering): safety_score AUROC = 0.54 (near random) — the safety filter does NOT discriminate selective from hemolytic AMPs. 5/8 hemolytic AMPs score safety=1.0, including melittin (HC50 ~ 5 ug/mL). The naive hydrophobic_fraction baseline (AUROC=0.81) significantly outperforms the designed safety_score. This is a documented failure, not a hidden one. The benchmark reveals the gap that the melittin blind spot hint in BENCHMARK_CARD.md previously only described qualitatively. The next loop should target safety_score improvement: lowering the hydrophobic_fraction threshold from 0.65 to ~0.50, and adding multi-residue hydrophobic face coverage analysis. 1417 tests pass, 1 skipped (pre-existing). Ruff clean on all new/modified files. --- Makefile | 6 +- docs/ARCHITECTURE.md | 2 +- docs/BENCHMARK_CARD.md | 23 +- docs/METRICS_CURRENT.md | 39 ++- docs/ROADMAP.md | 14 +- examples/validation/selectivity_panel.csv | 19 ++ src/openamp_foundry/benchmark/__init__.py | 1 + src/openamp_foundry/benchmark/selectivity.py | 245 ++++++++++++++++++ src/openamp_foundry/cli/commands/benchmark.py | 26 ++ src/openamp_foundry/cli/main.py | 25 +- tests/test_selectivity_benchmark.py | 214 +++++++++++++++ 11 files changed, 607 insertions(+), 7 deletions(-) create mode 100644 examples/validation/selectivity_panel.csv create mode 100644 src/openamp_foundry/benchmark/selectivity.py create mode 100644 tests/test_selectivity_benchmark.py diff --git a/Makefile b/Makefile index 9eaced33..39ad01ca 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-baseline bench-hidden-active generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter +.PHONY: help demo test lint ci clean bench-leakage bench-baseline bench-hidden-active generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict validate-selectivity external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -26,6 +26,7 @@ help: @echo " make validate-scoring AUROC on 95 AMP + 96 decoy (pipeline config)" @echo " make validate-scoring-phase3 AUROC with phase3.yaml config" @echo " make validate-scoring-strict AUROC with scrambled-decoy strict benchmark" + @echo " make validate-selectivity Selectivity benchmark: safety vs hemolytic AMPs" @echo " make bench-leakage Check for near-duplicates between candidates and refs" @echo " make bench-baseline Hidden-positive recovery benchmark (demo set)" @echo " make bench-hidden-active Hidden-positive recovery on mixed benchmark set" @@ -141,6 +142,9 @@ validate-scoring-strict: --benchmark-type strict \ --out outputs/validate_scoring_strict_report.json +validate-selectivity: + PYTHONPATH=src $(PYTHON) -c "from openamp_foundry.cli import main; main(['validate-selectivity', '--out', 'outputs/selectivity_report.json'])" + external-predict: pilot PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli external-predict \ --pilot-csv outputs/pilot_panel.csv \ diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a96484d3..503bdc7d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -39,7 +39,7 @@ The purpose of the added layers is not simulation theater. It is to improve whic | `openamp_foundry.scoring` | activity, safety, novelty, synthesis, ensemble scoring | | `openamp_foundry.selection` | ranking and diversity selection | | `openamp_foundry.evidence` | JSON certificate generation and validation | -| `openamp_foundry.benchmark` | leakage checks and evaluation scaffolding | +| `openamp_foundry.benchmark` | leakage checks, retrospective AUROC, selectivity benchmark, evaluation scaffolding | | `openamp_foundry.generators` | safe, bounded toy candidate generation | | `openamp_foundry.simulation` | membrane/selectivity/stability proxy modeling (scaffolded) | diff --git a/docs/BENCHMARK_CARD.md b/docs/BENCHMARK_CARD.md index 7433472a..a299288e 100644 --- a/docs/BENCHMARK_CARD.md +++ b/docs/BENCHMARK_CARD.md @@ -1,7 +1,7 @@ # Benchmark Card — OpenAMP Foundry v0.5.x > **Purpose:** Single-page summary of benchmark methodology, data, and metrics. -> **Last updated:** 2026-06-29 (PR #110 expanded benchmark) +> **Last updated:** 2026-07-01 (selectivity benchmark added) --- @@ -50,7 +50,7 @@ Confidence intervals: percentile bootstrap (2000 resamples, seed=0). | Bias | Impact | Mitigation | |------|--------|------------| | Helical-centric scorer | β-sheet AMPs (defensins) score below panel | Noted in METRICS_CURRENT.md | -| Melittin safety blind spot | Safety=1.0 despite strong hemolysis | Hemolysis assay mandatory for all | +| Melittin safety blind spot | Safety=1.0 despite strong hemolysis; 5/8 hemolytic AMPs score safety=1.0 (selectivity AUROC=0.54) | Hemolysis assay mandatory for all; safety_score redesign identified as improvement target | | Composition-matched scrambled AUROC < 0.5 | Model relies on composition > order; fails strict order-sensitivity test | OpenAMP is an **evidence-ranking tool**, not a validated sequence-order activity predictor. Composition-scrambled test confirms the model captures AMP-like composition, not sequence-order features. | | Near-seed generation only | Novel sequence space not explored | Acknowledged limitation in all docs | @@ -58,6 +58,24 @@ Confidence intervals: percentile bootstrap (2000 resamples, seed=0). OpenAMP performs well on standard decoys (AUROC 0.7832) but **fails the strict composition-scrambled benchmark (AUROC 0.4335, below random)**. This means the model primarily detects AMP-like amino acid composition rather than sequence-order-dependent antimicrobial features. This is expected for a physicochemical heuristic scorer and is appropriate for its intended use: **triage and ranking**, not deep biological prediction. +## Selectivity Benchmark + +> Tests whether safety features distinguish selective from hemolytic AMPs. +> Panel: 10 selective + 8 hemolytic AMPs (literature hemolysis classifications). +> Run: `make validate-selectivity` + +| Feature | AUROC | Verdict | +|---------|:-----:|---------| +| safety_score | 0.54 | **FAILED** — near random | +| selectivity_proxy | 0.61 | Weak | +| hydrophobic_fraction | 0.81 | Strong naive baseline | +| GRAVY | 0.76 | Strong naive baseline | +| hydrophobic_moment | 0.61 | Weak | + +**Finding:** safety_score does NOT discriminate selective from hemolytic AMPs. +5/8 hemolytic AMPs score safety=1.0 (melittin, mastoparan-X, PMAP-23, bombolitin-II, polybia-MP1). +hydrophobic_fraction is the best single discriminator. Improvement target identified. + ## Historical Baseline | Point | Set | Pipeline AUROC | Phase3 AUROC | @@ -71,4 +89,5 @@ OpenAMP performs well on standard decoys (AUROC 0.7832) but **fails the strict c | Date | Change | Author | |------|--------|--------| +| 2026-07-01 | Selectivity benchmark added — safety_score AUROC=0.54 | OpenAMP CI | | 2026-06-29 | Initial card — expanded benchmark (PR #110) | OpenAMP CI | diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index 7276868e..ec595396 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -3,7 +3,7 @@ > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-06-29 (PR #110 expanded benchmark) +> **Last updated:** 2026-07-01 (selectivity benchmark added) > **Pipeline version:** v0.5.x > **Branch:** main @@ -27,6 +27,43 @@ | Recall@43 | 0.4211 | 40/95 positives in top 43 | | Interpretation | **STRONG** | AUROC > 0.70 gate passed | +## Selectivity Benchmark Metrics + +> Tests whether pipeline safety features can distinguish selective AMPs (low hemolysis) +> from hemolytic AMPs. Reference panel: 10 selective + 8 hemolytic AMPs with literature +> hemolysis classifications. See `examples/validation/selectivity_panel.csv`. +> Run: `make validate-selectivity` + +| Feature | AUROC | Direction | Notes | +|---------|-------|-----------|-------| +| **safety_score** | **0.5437** | Higher = safer | **FAILED** — near random; does not discriminate selective from hemolytic | +| selectivity_proxy | 0.6125 | Higher = more selective | Weak signal above random | +| hydrophobic_fraction | 0.8063 | Higher = more hemolytic | **Best discriminator** — naive baseline outperforms designed safety score | +| hydrophobic_moment (μH) | 0.6125 | Higher = more hemolytic | Weak signal | +| charge_density_ph74 | 0.5437 | Higher = more hemolytic | Near random | +| GRAVY | 0.7625 | Higher = more hemolytic | Strong naive baseline | + +### Key findings + +1. **safety_score does NOT discriminate selective from hemolytic AMPs (AUROC=0.54, near random).** + Five of eight hemolytic AMPs receive safety_score=1.0 (maximum safety), including melittin + (HC50 ~ 5 ug/mL). The safety filter's thresholds for hydrophobic moment, charge density, and + hydrophobic fraction are too permissive for short helical peptides. + +2. **hydrophobic_fraction is the strongest single discriminator (AUROC=0.81).** Hemolytic AMPs + have higher hydrophobic fraction (mean 0.55 vs 0.35 for selective). The safety_score's + threshold (> 0.65) is set too high to catch this difference. + +3. **5 known blind spots identified:** melittin, mastoparan-X, PMAP-23, bombolitin-II, + polybia-MP1 all score safety=1.0 despite being literature-hemolytic. These are short + helical peptides where the hemolytic mechanism (oligomeric pore formation, toroidal + insertion) is not captured by 1D hydrophobic moment analysis. + +4. **Implication for candidate selection:** The current safety filter should NOT be trusted + as a hemolysis proxy. Wet-lab hemolysis assay remains mandatory for all candidates. + Improvement target: incorporate hydrophobic_fraction threshold lowering or add a + multi-residue hydrophobic face analysis to the safety score. + ### Historical baselines | Point | Benchmark | AUROC | Phase3 AUROC | Source | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 1607cea9..ac7a7a77 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -148,9 +148,21 @@ Implemented during the pre-wet-lab improvement loop (PRs #31–#54): - ~~write the first explicit simulator scope document: what OpenAMP will model and what it will not;~~ - ~~add membrane/selectivity/stability proxy interfaces with uncertainty fields;~~ -- benchmark candidate triage against a reference panel that includes selective AMPs, hemolytic positives, inactive peptides, and random controls; +- ~~benchmark candidate triage against a reference panel that includes selective AMPs, hemolytic positives, inactive peptides, and random controls;~~ **Done (2026-07-01):** `benchmark/selectivity.py` + `examples/validation/selectivity_panel.csv` — 10 selective + 8 hemolytic AMPs. Key finding: safety_score AUROC=0.54 (FAILED), hydrophobic_fraction AUROC=0.81 (best baseline). 5 blind spots identified. - require every proxy module to justify itself against cheap heuristic baselines before it affects selection. +## v1.1.x — Safety score improvement (identified by selectivity benchmark) + +The selectivity benchmark revealed that safety_score does not discriminate selective from +hemolytic AMPs (AUROC=0.54, near random). Improvement targets: + +- Lower hydrophobic_fraction threshold in safety_score from 0.65 to ~0.50 (captures the + selective/hemolytic boundary at hydrophobic_fraction ~0.45-0.55) +- Add a multi-residue hydrophobic face coverage metric (current μH threshold misses + short helical peptides where hemolysis arises from oligomeric pore formation) +- Re-run selectivity benchmark after any safety_score change to verify improvement +- Target: safety_score AUROC >= 0.70 on the selectivity panel + ## v2.x — Wet-lab compression engine - calibrate virtual assay layers on qualified assay outcomes; diff --git a/examples/validation/selectivity_panel.csv b/examples/validation/selectivity_panel.csv new file mode 100644 index 00000000..b844f805 --- /dev/null +++ b/examples/validation/selectivity_panel.csv @@ -0,0 +1,19 @@ +id,sequence,family,hemolysis_class,reference +SEL-MAG2,GIGKFLHSAKKFGKAFVGEIMNS,magainin,selective,Zasloff_1987_PNAS_HC50_gt_1mg_mL +SEL-CECA,KWKLFKKIEKVGQNIRDGIIKAGPAV,cecropin,selective,Steiner_1981_PNAS_low_hemolysis +SEL-BUF2,RAGLQFPVGRVHRLLRK,buforin,selective,Park_2000_JBC_non_lytic +SEL-HIS5,DSHAKRHHGYKRKFHEKHHSHRGY,histatin,selective,Oppenheim_1988_JBC_low_hemolysis +SEL-APIA,GNNRPVYIPQPRPPHPRI,apidaecin,selective,Casteels_1989_EMBO_low_hemolysis +SEL-DROS,GKPRPYSPRPTSHPRPIRV,drosocin,selective,Bulet_1993_JBC_low_hemolysis +SEL-AUR1,GLFDIIKKIAESF,aurein,selective,Rozek_2000_low_hemolysis +SEL-IND,ILPWKWPWWPWRR,indolicidin,selective,Selsted_1992_JBC_moderate_selectivity +SEL-PLEC,GFGCNGPWNEDDLRCHNHCKSIKGYKGGYCAKGGFVCKCY,plectasin,selective,Mygind_2005_Nature_low_hemolysis +SEL-TMPG,ILSKIFSLLKKVLP,temporin_G,selective,Simmaco_1996_low_hemolysis +HLY-MEL,GIGAVLKVLTTGLPALISWIKRKRQQ,melittin,hemolytic,Habermann_1972_HC50_5_ug_mL +HLY-MPX,INWKGIAAMAKKLL,mastoparan_x,hemolytic,Higashijima_1990_hemolytic +HLY-BMP28,GGLRSLGRKILRAWKKYGPIIVPIIRI,bmap28,hemolytic,Skerlavaj_1996_HC50_5_uM +HLY-PMAP23,RIIDLLWRVRRPQKPKFVTVWV,pmap23,hemolytic,Zanetti_1994_hemolytic +HLY-BOMB2,IKAILIKKILKGV,bombolitin_ii,hemolytic,Argiolas_1985_hemolytic +HLY-PMP1,IDWKKLLDAAKQIL,polybia_mp1,hemolytic,Souza_2005_hemolytic +HLY-LL37,LLGDFFRKSKEKIGKEFKRIVQRIKDFLRNLVPRTES,ll37,hemolytic,Turner_1998_hemolytic_high_conc +HLY-MPN,INLKALAALAKKLL,mastoparan,hemolytic,Hirai_1979_hemolytic diff --git a/src/openamp_foundry/benchmark/__init__.py b/src/openamp_foundry/benchmark/__init__.py index e69de29b..f2f17ba0 100644 --- a/src/openamp_foundry/benchmark/__init__.py +++ b/src/openamp_foundry/benchmark/__init__.py @@ -0,0 +1 @@ +"""Benchmark module: leakage checks, retrospective AUROC, selectivity benchmark, evaluation.""" diff --git a/src/openamp_foundry/benchmark/selectivity.py b/src/openamp_foundry/benchmark/selectivity.py new file mode 100644 index 00000000..602b2d9a --- /dev/null +++ b/src/openamp_foundry/benchmark/selectivity.py @@ -0,0 +1,245 @@ +"""Selectivity benchmark: can the pipeline distinguish selective AMPs from hemolytic AMPs? + +This benchmark tests a fundamentally different question from the retrospective AUROC: +instead of "can we distinguish AMPs from random sequences?", it asks "can we distinguish +AMPs that are safe (low hemolysis) from AMPs that are dangerous (high hemolysis)?" + +This is the critical benchmark for the safety-first objective. An AMP that kills bacteria +but also lyses RBCs is not a useful therapeutic candidate. If the pipeline cannot separate +selective from hemolytic AMPs, its safety filtering is not doing its job. + +Reference panel: ``examples/validation/selectivity_panel.csv`` +- 10 selective AMPs (literature-reported low hemolysis / HC50 > 100 uM or > 1 mg/mL) +- 8 hemolytic AMPs (literature-reported high hemolysis / HC50 < 50 uM) + +Key metrics: +- AUROC of safety_score for selective-vs-hemolytic discrimination +- AUROC of selectivity_proxy for the same task +- AUROC of naive baselines (hydrophobic_fraction, hydrophobic_moment, charge_density) +- Per-peptide score table with flags for known blind spots + +Literature basis for hemolysis classification: +- Melittin: HC50 ~ 5 ug/mL (Habermann 1972) — strongly hemolytic +- Mastoparan-X: hemolytic at >50 uM (Higashijima et al. 1990) +- BMAP-28: HC50 ~ 5 uM (Skerlavaj et al. 1996) — strongly hemolytic +- Magainin-2: HC50 > 1 mg/mL (~250 uM) (Zasloff 1987) — selective +- Cecropin-A: low hemolysis even at high concentration (Steiner et al. 1981) +- Buforin-II: non-lytic mechanism, no membrane disruption (Park et al. 2000) +- Histatin-5: low hemolysis, intracellular target (Oppenheim et al. 1988) +- Plectasin: low hemolysis, fungal defensin (Mygind et al. 2005) + +All hemolysis classifications are literature-derived. The benchmark is computational only; +it tests whether physicochemical proxies correlate with literature hemolysis classifications. +It does not predict hemolysis for novel sequences. +""" +from __future__ import annotations + +import csv +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass +class SelectivityRow: + id: str + sequence: str + family: str + hemolysis_class: str # "selective" or "hemolytic" + reference: str + safety_score: float + selectivity_proxy: float + hydrophobic_fraction: float + hydrophobic_moment: float + charge_density_ph74: float + gravy: float + label: int # 0 = selective, 1 = hemolytic (positive = dangerous) + + +@dataclass +class SelectivityResult: + n_selective: int + n_hemolytic: int + safety_auroc: float + selectivity_proxy_auroc: float + hydrophobic_fraction_auroc: float + hydrophobic_moment_auroc: float + charge_density_auroc: float + gravy_auroc: float + rows: list[SelectivityRow] = field(default_factory=list) + blind_spots: list[str] = field(default_factory=list) + verdict: str = "" + disclaimer: str = "" + + def to_dict(self) -> dict: + return { + "n_selective": self.n_selective, + "n_hemolytic": self.n_hemolytic, + "safety_auroc": self.safety_auroc, + "selectivity_proxy_auroc": self.selectivity_proxy_auroc, + "hydrophobic_fraction_auroc": self.hydrophobic_fraction_auroc, + "hydrophobic_moment_auroc": self.hydrophobic_moment_auroc, + "charge_density_auroc": self.charge_density_auroc, + "gravy_auroc": self.gravy_auroc, + "blind_spots": self.blind_spots, + "verdict": self.verdict, + "disclaimer": self.disclaimer, + "per_peptide": [ + { + "id": r.id, + "sequence": r.sequence, + "family": r.family, + "hemolysis_class": r.hemolysis_class, + "safety_score": r.safety_score, + "selectivity_proxy": r.selectivity_proxy, + "hydrophobic_fraction": r.hydrophobic_fraction, + "hydrophobic_moment": r.hydrophobic_moment, + "charge_density_ph74": r.charge_density_ph74, + "gravy": r.gravy, + } + for r in self.rows + ], + } + + +def _auc_wilcoxon(pos_scores: list[float], neg_scores: list[float]) -> float: + """AUROC via Wilcoxon-Mann-Whitney. Reuses the same method as retrospective.py. + + For selectivity: 'positive' = hemolytic (label=1, the dangerous class we want to detect). + A good safety score should be LOW for hemolytic peptides, so AUROC is computed as + P(safety_selective > safety_hemolytic) — i.e., safety_score should rank selectives higher. + We achieve this by treating selective as 'pos' and hemolytic as 'neg' for safety_score. + For other features (hydrophobic_fraction, gravy, hydrophobic_moment), higher values + correlate with hemolysis, so we treat hemolytic as 'pos' and selective as 'neg'. + """ + n_pos = len(pos_scores) + n_neg = len(neg_scores) + if n_pos == 0 or n_neg == 0: + return 0.5 + concordant = sum( + 1 for p in pos_scores for n in neg_scores if p > n + ) + 0.5 * sum( + 1 for p in pos_scores for n in neg_scores if p == n + ) + return concordant / (n_pos * n_neg) + + +def run_selectivity_benchmark( + panel_csv: str | Path = "examples/validation/selectivity_panel.csv", +) -> SelectivityResult: + """Run the selectivity benchmark on the reference panel. + + Returns a SelectivityResult with AUROC for each feature and per-peptide scores. + """ + from openamp_foundry.features.physchem import compute_features + from openamp_foundry.scoring.safety import safety_score + + rows: list[SelectivityRow] = [] + + with open(panel_csv, newline="", encoding="utf-8") as f: + for row in csv.DictReader(f): + seq = row["sequence"].strip().upper() + features = compute_features(seq) + safe = safety_score(features) + sel_proxy = features.get("selectivity_proxy", 1.0) + + hemolysis_class = row["hemolysis_class"].strip().lower() + label = 1 if hemolysis_class == "hemolytic" else 0 + + rows.append(SelectivityRow( + id=row["id"], + sequence=seq, + family=row["family"], + hemolysis_class=hemolysis_class, + reference=row["reference"], + safety_score=safe, + selectivity_proxy=sel_proxy, + hydrophobic_fraction=features["hydrophobic_fraction"], + hydrophobic_moment=features.get("hydrophobic_moment", 0.0), + charge_density_ph74=features.get("charge_density_ph74", 0.0), + gravy=features.get("gravy", 0.0), + label=label, + )) + + selective = [r for r in rows if r.label == 0] + hemolytic = [r for r in rows if r.label == 1] + + # For safety_score and selectivity_proxy: higher = safer = should rank selectives higher. + # AUROC = P(score_selective > score_hemolytic). + sel_safety = [r.safety_score for r in selective] + hly_safety = [r.safety_score for r in hemolytic] + safety_auroc = round(_auc_wilcoxon(sel_safety, hly_safety), 4) + + sel_proxy = [r.selectivity_proxy for r in selective] + hly_proxy = [r.selectivity_proxy for r in hemolytic] + proxy_auroc = round(_auc_wilcoxon(sel_proxy, hly_proxy), 4) + + # For hydrophobic_fraction, hydrophobic_moment, gravy: higher = more hemolytic. + # AUROC = P(score_hemolytic > score_selective). + hly_hfrac = [r.hydrophobic_fraction for r in hemolytic] + sel_hfrac = [r.hydrophobic_fraction for r in selective] + hfrac_auroc = round(_auc_wilcoxon(hly_hfrac, sel_hfrac), 4) + + hly_muh = [r.hydrophobic_moment for r in hemolytic] + sel_muh = [r.hydrophobic_moment for r in selective] + muh_auroc = round(_auc_wilcoxon(hly_muh, sel_muh), 4) + + hly_gravy = [r.gravy for r in hemolytic] + sel_gravy = [r.gravy for r in selective] + gravy_auroc = round(_auc_wilcoxon(hly_gravy, sel_gravy), 4) + + # Charge density: ambiguous — high charge can mean either strong AMP (selective) + # or non-specific disruption. Test both directions and report the one that + # discriminates (higher = hemolytic). + hly_charge = [r.charge_density_ph74 for r in hemolytic] + sel_charge = [r.charge_density_ph74 for r in selective] + charge_auroc = round(_auc_wilcoxon(hly_charge, sel_charge), 4) + + # Identify blind spots: hemolytic peptides with safety_score >= 0.90 + blind_spots: list[str] = [] + for r in hemolytic: + if r.safety_score >= 0.90: + blind_spots.append( + f"{r.id} ({r.family}): safety_score={r.safety_score} but " + f"literature-hemolytic — known blind spot" + ) + # Also flag selective peptides with low safety_score (false positives) + for r in selective: + if r.safety_score < 0.70: + blind_spots.append( + f"{r.id} ({r.family}): safety_score={r.safety_score} but " + f"literature-selective — false alarm" + ) + + # Verdict: safety_score should achieve AUROC > 0.65 to be considered useful + # for selectivity discrimination. Below 0.55 means it's worse than random. + if safety_auroc >= 0.70: + verdict = "STRONG — safety_score discriminates selective from hemolytic" + elif safety_auroc >= 0.65: + verdict = "MODERATE — safety_score has partial selectivity signal" + elif safety_auroc >= 0.55: + verdict = "WEAK — safety_score barely above random for selectivity" + else: + verdict = "FAILED — safety_score does NOT discriminate selective from hemolytic" + + disclaimer = ( + "This benchmark tests whether physicochemical proxies correlate with " + "literature hemolysis classifications on a small reference panel (n=18). " + "It does NOT predict hemolysis for novel sequences. Wet-lab hemolysis " + "assay remains mandatory for all candidates. The panel is small and " + "class-imbalanced; results are indicative, not definitive." + ) + + return SelectivityResult( + n_selective=len(selective), + n_hemolytic=len(hemolytic), + safety_auroc=safety_auroc, + selectivity_proxy_auroc=proxy_auroc, + hydrophobic_fraction_auroc=hfrac_auroc, + hydrophobic_moment_auroc=muh_auroc, + charge_density_auroc=charge_auroc, + gravy_auroc=gravy_auroc, + rows=rows, + blind_spots=blind_spots, + verdict=verdict, + disclaimer=disclaimer, + ) diff --git a/src/openamp_foundry/cli/commands/benchmark.py b/src/openamp_foundry/cli/commands/benchmark.py index 8afd7be6..bd6c4a88 100644 --- a/src/openamp_foundry/cli/commands/benchmark.py +++ b/src/openamp_foundry/cli/commands/benchmark.py @@ -81,3 +81,29 @@ def _run_validate_scoring(args: argparse.Namespace) -> int: return 0 + + +def _run_validate_selectivity(args: argparse.Namespace) -> int: + import json as _json + from openamp_foundry.benchmark.selectivity import run_selectivity_benchmark + from openamp_foundry.utils.io import write_json + + result = run_selectivity_benchmark(panel_csv=args.panel_csv) + d = result.to_dict() + if args.out: + write_json(args.out, d) + summary = { + "status": "ok", + "n_selective": result.n_selective, + "n_hemolytic": result.n_hemolytic, + "safety_auroc": result.safety_auroc, + "selectivity_proxy_auroc": result.selectivity_proxy_auroc, + "hydrophobic_fraction_auroc": result.hydrophobic_fraction_auroc, + "hydrophobic_moment_auroc": result.hydrophobic_moment_auroc, + "charge_density_auroc": result.charge_density_auroc, + "gravy_auroc": result.gravy_auroc, + "verdict": result.verdict, + "blind_spots": result.blind_spots, + } + print(_json.dumps(summary, indent=2)) + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index f4dc937e..35e4cef6 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -1,7 +1,7 @@ from __future__ import annotations from openamp_foundry.cli.commands.core import _run_generate_batch -from openamp_foundry.cli.commands.benchmark import _run_bench, _run_validate_scoring +from openamp_foundry.cli.commands.benchmark import _run_bench, _run_validate_scoring, _run_validate_selectivity from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc @@ -191,6 +191,26 @@ def build_parser() -> argparse.ArgumentParser: help="Optional JSON output path.", ) + validate_selectivity = sub.add_parser( + "validate-selectivity", + help=( + "Selectivity benchmark: can the pipeline distinguish selective AMPs " + "(low hemolysis) from hemolytic AMPs? Tests safety_score, selectivity_proxy, " + "and naive baselines against a literature-curated reference panel. " + "AUROC > 0.65 = safety filter has useful selectivity signal." + ), + ) + validate_selectivity.add_argument( + "--panel-csv", + default="examples/validation/selectivity_panel.csv", + help="CSV of selective/hemolytic AMPs with 'hemolysis_class' column.", + ) + validate_selectivity.add_argument( + "--out", + required=False, + help="Optional JSON output path.", + ) + external_predict = sub.add_parser( "external-predict", help=( @@ -518,6 +538,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "validate-scoring": return _run_validate_scoring(args) + if args.command == "validate-selectivity": + return _run_validate_selectivity(args) + if args.command == "external-predict": return _run_external_predict(args) diff --git a/tests/test_selectivity_benchmark.py b/tests/test_selectivity_benchmark.py new file mode 100644 index 00000000..d7cf3f0a --- /dev/null +++ b/tests/test_selectivity_benchmark.py @@ -0,0 +1,214 @@ +"""Selectivity benchmark tests — Phase 2.5 requirement. + +Per AGENTS.md Phase 2.5: "Membrane proxy benchmark — Distinguishes bacterial-selective +vs clearly hemolytic reference peptides better than naive heuristics." + +These tests verify that: +1. The selectivity benchmark runs on the reference panel. +2. The benchmark produces correct AUROC values for all features. +3. Known blind spots (melittin safety=1.0 despite hemolysis) are detected and reported. +4. At least one feature discriminates selective from hemolytic above 0.60 AUROC. +5. The benchmark output includes per-peptide scores and baseline comparisons. +6. The CLI command produces valid output. + +CURRENT FINDING (2026-07-01): +safety_score AUROC = 0.54 (near random) — the safety filter does NOT discriminate +selective from hemolytic AMPs. hydrophobic_fraction AUROC = 0.81 (strong baseline). +This gap is documented as a known issue; the test verifies the benchmark honestly +reveals it rather than hiding it. + +All results are computational proxies. No biological hemolysis is measured. +The benchmark tests correlation with literature hemolysis classifications only. +""" +from __future__ import annotations + +import csv +import json +from pathlib import Path + +import pytest + +from openamp_foundry.benchmark.selectivity import ( + SelectivityResult, + _auc_wilcoxon, + run_selectivity_benchmark, +) + + +PANEL_CSV = "examples/validation/selectivity_panel.csv" + + +class TestSelectivityPanel: + """Verify the reference panel data integrity.""" + + def test_panel_exists(self): + assert Path(PANEL_CSV).exists(), "Selectivity panel CSV not found" + + def test_panel_has_expected_columns(self): + with open(PANEL_CSV) as f: + reader = csv.DictReader(f) + assert "id" in reader.fieldnames + assert "sequence" in reader.fieldnames + assert "hemolysis_class" in reader.fieldnames + + def test_panel_has_both_classes(self): + with open(PANEL_CSV) as f: + rows = list(csv.DictReader(f)) + classes = {r["hemolysis_class"] for r in rows} + assert "selective" in classes + assert "hemolytic" in classes + + def test_panel_has_at_least_8_per_class(self): + with open(PANEL_CSV) as f: + rows = list(csv.DictReader(f)) + n_sel = sum(1 for r in rows if r["hemolysis_class"] == "selective") + n_hly = sum(1 for r in rows if r["hemolysis_class"] == "hemolytic") + assert n_sel >= 8, f"Need at least 8 selective AMPs, got {n_sel}" + assert n_hly >= 5, f"Need at least 5 hemolytic AMPs, got {n_hly}" + + def test_all_sequences_are_valid_amino_acids(self): + with open(PANEL_CSV) as f: + rows = list(csv.DictReader(f)) + valid_aa = set("ACDEFGHIKLMNPQRSTVWY") + for row in rows: + seq = row["sequence"].upper() + assert all(aa in valid_aa for aa in seq), f"Invalid AA in {row['id']}: {seq}" + + +class TestSelectivityBenchmark: + """Core benchmark tests.""" + + @pytest.fixture + def result(self): + return run_selectivity_benchmark(PANEL_CSV) + + def test_returns_selectivity_result(self, result): + assert isinstance(result, SelectivityResult) + + def test_correct_counts(self, result): + assert result.n_selective > 0 + assert result.n_hemolytic > 0 + assert result.n_selective + result.n_hemolytic == len(result.rows) + + def test_at_least_one_feature_discriminates(self, result): + """At least one feature must achieve AUROC > 0.60 for selectivity. + + If none do, the benchmark panel or the pipeline is fundamentally broken. + Currently hydrophobic_fraction (AUROC ~0.81) carries the signal. + """ + all_aurocs = [ + result.safety_auroc, + result.selectivity_proxy_auroc, + result.hydrophobic_fraction_auroc, + result.hydrophobic_moment_auroc, + result.charge_density_auroc, + result.gravy_auroc, + ] + best = max(all_aurocs) + assert best > 0.60, ( + f"No feature achieves AUROC > 0.60 (best={best:.4f}). " + "The selectivity benchmark panel may be too small or the pipeline " + "has no selectivity signal at all." + ) + + def test_safety_auroc_reported_honestly(self, result): + """safety_score AUROC is reported regardless of whether it passes or fails. + + Current value: ~0.54 (near random). This test verifies the value is + reported correctly and is in [0, 1]. It does NOT assert the score is good. + The safety_score's failure to discriminate is a documented finding. + """ + assert 0.0 <= result.safety_auroc <= 1.0 + + def test_per_peptide_scores_present(self, result): + assert len(result.rows) > 0 + row = result.rows[0] + assert hasattr(row, "safety_score") + assert hasattr(row, "selectivity_proxy") + assert hasattr(row, "hydrophobic_fraction") + assert hasattr(row, "hydrophobic_moment") + + def test_to_dict_produces_valid_json(self, result): + d = result.to_dict() + json.dumps(d) + assert "safety_auroc" in d + assert "per_peptide" in d + + def test_blind_spots_detected(self, result): + """Known blind spots (hemolytic with high safety_score) should be flagged. + + Melittin is the canonical blind spot: safety=1.0 despite HC50 ~ 5 ug/mL. + At least 3 hemolytic peptides should appear in blind_spots (melittin, + mastoparan-X, and at least one more). + """ + assert len(result.blind_spots) >= 3, ( + f"Expected at least 3 blind spots (hemolytic with safety >= 0.90), " + f"got {len(result.blind_spots)}: {result.blind_spots}" + ) + blind_spot_text = " ".join(result.blind_spots).lower() + assert "melittin" in blind_spot_text + + def test_verdict_is_informative(self, result): + assert len(result.verdict) > 10 + assert any(w in result.verdict for w in ["STRONG", "MODERATE", "WEAK", "FAILED"]) + + def test_disclaimer_present(self, result): + assert "not predict" in result.disclaimer.lower() + assert "wet-lab" in result.disclaimer.lower() or "wet lab" in result.disclaimer.lower() + + def test_baseline_aurocs_computed(self, result): + """All baseline AUROCs should be present and in [0, 1].""" + for auroc in [result.hydrophobic_fraction_auroc, result.hydrophobic_moment_auroc, + result.charge_density_auroc, result.gravy_auroc]: + assert 0.0 <= auroc <= 1.0 + + def test_hydrophobic_fraction_beats_safety(self, result): + """hydrophobic_fraction should outperform safety_score for selectivity. + + This is a key finding: the naive hydrophobic_fraction baseline (AUROC ~0.81) + significantly outperforms the designed safety_score (AUROC ~0.54). + If this test fails, either safety_score was improved (good — update test) + or hydrophobic_fraction lost discriminative power (investigate). + """ + assert result.hydrophobic_fraction_auroc > result.safety_auroc, ( + f"hydrophobic_fraction AUROC ({result.hydrophobic_fraction_auroc}) should " + f"exceed safety_score AUROC ({result.safety_auroc}). If safety_score was " + f"improved to beat this baseline, update this test." + ) + + +class TestAucWilcoxonSelectivity: + """Unit tests for the AUROC computation in selectivity context.""" + + def test_perfect_separation(self): + pos = [0.9, 0.8, 0.7] + neg = [0.4, 0.3, 0.2] + assert _auc_wilcoxon(pos, neg) == pytest.approx(1.0) + + def test_worst_case(self): + pos = [0.2, 0.3, 0.4] + neg = [0.7, 0.8, 0.9] + assert _auc_wilcoxon(pos, neg) == pytest.approx(0.0) + + def test_random_is_half(self): + pos = [0.5, 0.5, 0.5] + neg = [0.5, 0.5, 0.5] + assert _auc_wilcoxon(pos, neg) == pytest.approx(0.5) + + def test_empty_returns_half(self): + assert _auc_wilcoxon([], [0.5, 0.6]) == pytest.approx(0.5) + + +class TestSelectivityCLI: + """CLI integration test for validate-selectivity command.""" + + def test_cli_runs_and_produces_json(self, tmp_path): + from openamp_foundry.cli import main + out_json = tmp_path / "selectivity_result.json" + rc = main(["validate-selectivity", "--out", str(out_json)]) + assert rc == 0 + assert out_json.exists() + data = json.loads(out_json.read_text()) + assert "safety_auroc" in data + assert "per_peptide" in data + assert len(data["per_peptide"]) > 0