diff --git a/CHANGELOG.md b/CHANGELOG.md index a3130686..8bace7b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,37 @@ All notable changes to OpenAMP Foundry are documented here. --- +## [Unreleased — Within-AMP Selectivity Benchmark] — 2026-07-01 + +### v0.5.9 — Within-AMP Selectivity Benchmark + +- **Hemolysis reference dataset**: 42 known AMPs with literature HC50 values + (`examples/validation/hemolysis_reference.csv`): 14 HEMOLYTIC (HC50 < 25 µg/mL), + 21 SELECTIVE (HC50 >= 100 µg/mL), 7 BORDER (25-100 µg/mL) +- **`run_selectivity_benchmark()`** in `benchmark/retrospective.py`: tests whether + pipeline scorers can distinguish hemolytic from selective AMPs — the within-AMP + ranking task that the expert ablation identified as the correct test for + safety/selectivity/synthesis components +- **`bench selectivity` CLI command** + `make bench-selectivity` Makefile target +- **18 new tests** in `test_selectivity_benchmark.py`; 1453 total tests + +**Key findings:** + +1. Safety scorer FAILS hemolysis detection (detection AUROC=0.3844, CI lo=0.26). + All 14 hemolytic AMPs score safety >= 0.8. Melittin blind spot confirmed + quantitatively. +2. Selectivity proxy FAILS hemolysis detection (detection AUROC=0.4133, CI lo=0.28). + Charge/GRAVY heuristic insufficient. +3. Synthesis feasibility is the only significant hemolysis risk detector + (detection AUROC=0.8027, CI lo=0.63) — incidental correlation via cysteine + content and repeat runs in hemolytic AMPs. +4. Expert composite is better than ensemble on hemolysis detection (0.5119 vs + 0.3486) but not significant (CI includes 0.5) at n=14 vs n=21. +5. Activity scorer is anti-selective (detection AUROC=0.34): ranks hemolytic + AMPs higher due to stronger amphipathic helices. Ensemble inherits this bias. + +--- + ## [Unreleased — Pre-synthesis Quality Sprint] — 2026-06 **Status:** Pipeline frozen for synthesis batch ordering. All changes below were quality diff --git a/Makefile b/Makefile index b170e83b..2cf76853 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter +.PHONY: help demo test lint ci clean bench-leakage bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-selectivity generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -30,6 +30,7 @@ help: @echo " make bench-baseline Hidden-positive recovery benchmark (demo set)" @echo " make bench-cluster-split Cluster-aware bootstrap CI (de-inflates near-duplicates)" @echo " make bench-expert-ablation Expert composite vs ensemble ablation (honesty check)" + @echo " make bench-selectivity Within-AMP selectivity (hemolytic vs selective)" @echo " make bench-hidden-active Hidden-positive recovery on mixed benchmark set" @echo "" @echo " make wave0-5-gate-check Run Wave 0.5 gates W0.5-1 through W0.5-7" @@ -148,6 +149,11 @@ bench-expert-ablation: --decoy-csv examples/validation/random_background.csv --out outputs/expert_ablation_report.json +bench-selectivity: + PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench selectivity + --hemolysis-csv examples/validation/hemolysis_reference.csv + --out outputs/selectivity_benchmark_report.json + validate-scoring-strict: PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli validate-scoring \ --amp-csv examples/validation/known_amps.csv \ diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 594c2a44..b0f709ba 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -39,7 +39,7 @@ The purpose of the added layers is not simulation theater. It is to improve whic | `openamp_foundry.scoring` | activity, safety, novelty, synthesis, ensemble scoring | | `openamp_foundry.selection` | ranking and diversity selection | | `openamp_foundry.evidence` | JSON certificate generation and validation | -| `openamp_foundry.benchmark` | leakage checks, cluster-split benchmark, expert ablation benchmark, and evaluation scaffolding | +| `openamp_foundry.benchmark` | leakage checks, cluster-split benchmark, expert ablation benchmark, within-AMP selectivity benchmark, and evaluation scaffolding | | `openamp_foundry.generators` | safe, bounded toy candidate generation | | `openamp_foundry.simulation` | membrane/selectivity/stability proxy modeling (scaffolded) | diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 699c8060..23e9e0e9 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -25,6 +25,7 @@ Common artifacts: | Hidden-positive recovery | Check whether known actives are ranked high | | Toxicity down-ranking | Ensure predicted risky candidates are penalized | | Novelty stress test | Avoid near-clones of references | +| Within-AMP selectivity | Test whether scorers distinguish hemolytic from selective AMPs | ## Minimum report fields diff --git a/docs/EXPERT_REVIEW_PACK.md b/docs/EXPERT_REVIEW_PACK.md index d28f0e24..e08d7f91 100644 --- a/docs/EXPERT_REVIEW_PACK.md +++ b/docs/EXPERT_REVIEW_PACK.md @@ -295,7 +295,7 @@ The expert reviewer must be aware of these limitations: |-----------|--------| | Level 0–2 evidence only | Scores are physicochemical heuristics, not validated ML predictions | | AUROC = 0.7832 on n=191 (expanded) | CI₉₅: 0.72–0.84; 52 new AMPs from 12 taxonomic classes now included (defensins, proline-rich, lantibiotics) — more honest estimate but broader AMP classes still challenge the helic-centric scorer | -| Safety model blind spot | Melittin scores Safety=1.0; curvature-mediated hemolysis not captured; hemolysis assay mandatory | +| Safety model blind spot | Melittin scores Safety=1.0; confirmed quantitatively in v0.5.9 selectivity benchmark: safety scorer detection AUROC=0.3844 (14 hemolytic vs 21 selective AMPs); all 14 hemolytic AMPs score safety >= 0.8; hemolysis assay mandatory | | SEED-008 high disagreement (0.41–0.44) | Boman index penalises Trp; activity_likeness more appropriate for Trp-rich sequences | | Serum stability model calibrated for 18–30 AA | SEED-003 (11 AA), SEED-007 (11 AA), and SEED-008 (13 AA) scores may underestimate actual stability (short-peptide steric/flexibility effects) | | Pro-rich serum stability not in model | SEED-009 model scores (0.572) expected to underestimate actual stability; Pro-X bond resistance is mechanistic | diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index bfd85810..2de2933d 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -224,16 +224,93 @@ frequencies tend to have more moderate biophysical properties. only. The expert composite's selectivity, safety, and synthesis components are designed for within-AMP candidate differentiation, not for separating AMPs from non-AMPs. A within-AMP ranking benchmark (comparing selective vs hemolytic AMPs) -would be a more appropriate test for those components and is a candidate for the -next loop. +has been added in v0.5.9 (see Within-AMP Selectivity Benchmark section below). --- + +## Within-AMP Selectivity Benchmark (v0.5.x — added 2026-07-01) + +> The expert ablation benchmark found that safety, synthesis, and serum stability are +> anti-signal for AMP-vs-decoy discrimination. But those scorers were designed for +> *within-AMP ranking*: distinguishing hemolytic AMPs from selective AMPs. This benchmark +> tests them on that intended task. +> +> Run: `make bench-selectivity` + +**Dataset:** 42 known AMPs with literature HC50 values (hemolysis_reference.csv) + +| Class | HC50 threshold | Count | +|-------|:--------------:|:-----:| +| HEMOLYTIC | < 25 µg/mL | 14 | +| SELECTIVE | >= 100 µg/mL | 21 | +| BORDER (excluded from AUROC) | 25-100 µg/mL | 7 | + +**Task:** Can pipeline scorers distinguish hemolytic AMPs from selective AMPs? +For safety/selectivity scorers, the correct direction is: hemolytic AMPs score *lower* +(less safe, less selective). We report "hemolysis detection AUROC" where higher = better +risk detection (1 - raw AUROC for safety-type scorers). + +### Per-score hemolysis detection AUROC + +| Score | Detection AUROC | CI₉₅ | Significant? | Verdict | +|-------|:--------------:|:----:|:------------:|---------| +| synthesis | 0.8027 | 0.63-0.95 | **YES** | Synthesis difficulty correlates with hemolysis — hemolytic AMPs are harder to synthesize | +| boman_activity | 0.6837 | 0.49-0.85 | No (CI lo < 0.5) | Weak trend: hemolytic AMPs have lower Boman activity | +| serum_stability | 0.6020 | 0.40-0.80 | No (CI lo < 0.5) | Weak trend: hemolytic AMPs less serum-stable | +| expert_composite | 0.5119 | 0.31-0.71 | No (CI lo < 0.5) | Better than ensemble but not significant | +| hinge_selectivity | 0.4456 | 0.24-0.64 | No | No selectivity signal from hinge detection | +| selectivity_proxy | 0.4133 | 0.28-0.55 | No | **FAILS** — charge/GRAVY does not capture hemolysis | +| safety | 0.3844 | 0.26-0.52 | No | **FAILS** — confirms melittin blind spot | +| activity | 0.3401 | 0.16-0.52 | No | Activity scorer ranks hemolytic AMPs *higher* (anti-selective) | +| ensemble | 0.3486 | 0.17-0.54 | No | Ensemble inherits activity scorer's anti-selective bias | + +**Key findings:** + +1. **The safety scorer does NOT detect hemolysis** (detection AUROC = 0.3844, CI lo = 0.26). + This confirms the expert ablation's prediction and the previously documented melittin + blind spot. All 14 hemolytic AMPs in the reference set score safety >= 0.8 — the scorer + cannot distinguish them from selective AMPs. + +2. **The selectivity proxy does NOT detect hemolysis** (detection AUROC = 0.4133, CI lo = 0.28). + The charge/GRAVY heuristic is insufficient for capturing hemolysis risk. Hemolytic AMPs + like melittin and protegrin have optimal charge (+2 to +7) and moderate GRAVY, so the + proxy assigns them high selectivity scores. + +3. **Synthesis feasibility is the only significant risk detector** (detection AUROC = 0.8027, + CI lo = 0.63). Hemolytic AMPs tend to be harder to synthesize: they have more cysteines + (protegrins, tachyplesins), repeat runs, and hydrophobic segments. This is an incidental + correlation, not a designed safety feature — but it means the synthesis gate provides + partial hemolysis filtering as a side effect. + +4. **The activity scorer is anti-selective** (detection AUROC = 0.34): it ranks hemolytic + AMPs *higher* than selective AMPs. This is expected: hemolytic AMPs like melittin have + strong amphipathic helices, high hydrophobic moment, and high charge — exactly the + features the activity scorer rewards. The ensemble inherits this bias. + +5. **The expert composite is better than the ensemble** on hemolysis detection (0.5119 vs + 0.3486) but not significantly so (CI includes 0.5). The added selectivity and safety + components partially offset the activity scorer's anti-selective bias, but not enough + to reach significance at n=14 vs n=21. + +**Honest limitation:** HC50 values are approximate literature values with high inter-assay +variability (RBC source, buffer, incubation time, concentration range). The binary +thresholds (25 / 100 µg/mL) are coarse. A larger reference set with standardized HC50 +measurements would tighten the CIs and might flip some near-zero results to significant. +The current sample size (14 vs 21) is too small for confident conclusions on any score +with CI lower bound below 0.5. + +**Implication for the pipeline:** Hemolysis remains unpredictable by the current +physicochemical scorers. The melittin blind spot is confirmed quantitatively. Hemolysis +must be assayed experimentally for every candidate regardless of safety or selectivity +score. The synthesis gate provides partial indirect filtering but should not be relied +upon as a hemolysis predictor. + ## Test Suite | Metric | Value | |--------|-------| -| Total tests | 1435 | +| Total tests | 1453 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | @@ -262,4 +339,5 @@ next loop. | 2026-06-29 | External predictor results filled from wave05_combined_consensus.csv; all 7 gates PASS | OpenAMP Wave 0.5 | | 2026-06-29 | Wave 0.5 scaffold diversification — 24-candidate Wave 1 panel across 14 families | OpenAMP Wave 0.5 | | 2026-07-01 | Expert ablation benchmark added: expert composite AUROC 0.736 vs ensemble 0.7832 (delta −0.0472); 3 components anti-signal; ensemble remains primary gate | OpenAMP loop | +| 2026-07-01 | Within-AMP selectivity benchmark added: safety scorer FAILS hemolysis detection (AUROC=0.3844); synthesis is only significant risk detector (AUROC=0.8027); expert composite better than ensemble but not significant (0.5119 vs 0.3486) | OpenAMP loop | | 2026-06-29 | Initial — expanded benchmark (PR #110) | OpenAMP CI | diff --git a/docs/REVIEWER_SUMMARY.md b/docs/REVIEWER_SUMMARY.md index d50d8a34..2f277912 100644 --- a/docs/REVIEWER_SUMMARY.md +++ b/docs/REVIEWER_SUMMARY.md @@ -77,7 +77,7 @@ assay is mandatory for all candidates. | AUROC 0.7832 — 22% of benchmark pairs misranked | ~1 in 5 candidate ranking decisions are wrong; wet-lab is the judge | | No 3D structure or docking | Helical assumption may misclassify non-helical mechanisms | | Near-seed generation only | Novel sequence space not explored by de novo generation | -| Melittin safety blind spot | Hemolysis must be assayed, cannot be predicted | +| Melittin safety blind spot | Hemolysis must be assayed — confirmed quantitatively: safety scorer detection AUROC=0.3844 on 14 hemolytic vs 21 selective AMPs (v0.5.9 selectivity benchmark) | | No external predictor ensemble yet | CAMPR4/AMPScanner/dbAMP/AntiCP2/Macrel web submission checklist exists at `outputs/external_predict_checklist.md` but results not yet obtained | | Probability table uncalibrated | All "P(≥1 active) ≈ 92–97%" statements are upper bounds; honest estimate 55–80% for any hit, 35–60% for novel family | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 77be9e73..4d7988c6 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -142,10 +142,24 @@ Implemented during the pre-wet-lab improvement loop (PRs #31–#54): - `bench expert-ablation` CLI command + `make bench-expert-ablation` target - Per-component AUROC attribution: identifies signal-bearing (activity 0.814, selectivity 0.773), near-zero (hinge, novelty, boman, motif), and anti-signal (safety 0.349, serum_stability 0.223, synthesis 0.423) components - **Key finding:** Expert composite AUROC 0.736 < ensemble AUROC 0.783 (delta −0.047). The added complexity does NOT improve AMP-vs-decoy discrimination. Three components are anti-signal because real AMPs have extreme biophysical properties that safety/stability/synthesis scorers penalise. -- Ensemble remains primary synthesis gate. Expert composite may add value for within-AMP ranking (not yet demonstrated). +- Ensemble remains primary synthesis gate. Expert composite may add value for within-AMP ranking (tested in v0.5.9 — see selectivity benchmark; expert composite detection AUROC 0.5119 vs ensemble 0.3486, but not significant at n=14 vs n=21). - `make bench-cluster-split` Makefile target added (was missing despite CLI command existing) - 14 new tests in `test_expert_ablation.py`; 1435 total tests +## v0.5.9 — Within-AMP Selectivity Benchmark ✓ (2026-07-01) + +- `run_selectivity_benchmark()` in `benchmark/retrospective.py` — tests whether pipeline scorers can distinguish hemolytic AMPs (HC50 < 25 µg/mL) from selective AMPs (HC50 >= 100 µg/mL) +- `bench selectivity` CLI command + `make bench-selectivity` target +- Hemolysis reference dataset: 42 known AMPs with literature HC50 values (`examples/validation/hemolysis_reference.csv`) +- Per-score hemolysis detection AUROC: identifies the only significant risk detector (synthesis, AUROC=0.8027, CI: 0.63-0.95) and confirms the safety scorer blind spot (detection AUROC=0.3844) +- **Key finding:** Safety scorer FAILS to detect hemolysis (detection AUROC=0.3844, CI lo=0.26). All 14 hemolytic AMPs score safety >= 0.8. The melittin blind spot is confirmed quantitatively — 1D hydrophobic moment cannot capture hemolysis mechanisms like curvature-mediated lysis or beta-sheet pore formation. +- **Key finding:** Selectivity proxy FAILS to detect hemolysis (detection AUROC=0.4133, CI lo=0.28). The charge/GRAVY heuristic is insufficient — hemolytic AMPs like melittin have optimal charge and moderate GRAVY. +- **Key finding:** Synthesis feasibility is the only significant hemolysis risk detector (detection AUROC=0.8027, CI lo=0.63). This is an incidental correlation: hemolytic AMPs tend to have more cysteines, repeat runs, and hydrophobic segments that make them harder to synthesize. The synthesis gate provides partial indirect hemolysis filtering. +- **Key finding:** Expert composite is better than ensemble on hemolysis detection (0.5119 vs 0.3486) but not significant (CI includes 0.5). The added selectivity/safety components partially offset the activity scorer's anti-selective bias. +- **Key finding:** Activity scorer is anti-selective (detection AUROC=0.34): it ranks hemolytic AMPs *higher* because they have stronger amphipathic helices. The ensemble inherits this bias. +- Blind spots: 14 hemolytic AMPs with safety >= 0.8 (melittin, indolicidin, protegrins, tachyplesins, piscidin-1, arenicin, polyphemusin, gramicidin-S-like, BMAP fragment) +- 18 new tests in `test_selectivity_benchmark.py`; 1453 total tests + ## v1.0 — Validated dry-lab-to-wet-lab loop - independently reviewed assay batch (expert_review.yml GitHub issue template); diff --git a/examples/validation/hemolysis_reference.csv b/examples/validation/hemolysis_reference.csv new file mode 100644 index 00000000..eed25f60 --- /dev/null +++ b/examples/validation/hemolysis_reference.csv @@ -0,0 +1,43 @@ +id,sequence,family,hc50_ugml,hemolysis_class,reference +REF-MEL-001,GIGAVLKVLTTGLPALISWIKRKRQQ,melittin,1.5,HEMOLYTIC,Habermann_1972 +REF-MAG-001,GIGKFLHSAKKFGKAFVGEIMNS,magainin_2,100,SELECTIVE,Zasloff_1987_PNAS +REF-MAG-002,GIGKFLHSAGKFGKAFVGEIMKS,magainin_2_variant,100,SELECTIVE,Zasloff_1987_PNAS +REF-PEX-001,GIGKFLKKAKKFGKAFVKILKK,pexiganan,100,SELECTIVE,Ge_1999_Antimicrob +REF-CEC-001,KWKLFKKIEKVGQNIRDGIIKAGPAV,cecropin_A,200,SELECTIVE,Steiner_1981_PNAS +REF-CEC-002,KWKIFKKIEKVGRNVRDGIIKAGP,cecropin_B,200,SELECTIVE,Hultmark_1980 +REF-LL37-002,LLGDFFRKSKEKIGKEFKRIVQRIKDFLRNLVPRTES,LL_37,100,SELECTIVE,Gudmundsson_1996_PNAS +REF-IND-001,ILPWKWPWWPWRR,indolicidin,10,MODERATE,Selsted_1992_J_Biol_Chem +REF-IND-002,ILPWKWPWWPWRRRR,indolicidin_analog,5,MODERATE,Falla_1996_J_Biol_Chem +REF-TMP-001,FLPLIGRVLSGIL,temporin_A,50,MODERATE,Mangoni_2001_FEBS +REF-TMP-002,FVQWFSKFLGRIL,temporin_L,40,MODERATE,Mangoni_2001_FEBS +REF-TMP-003,LLPIVGNLLKSLL,temporin_B,100,SELECTIVE,Simmaco_1996 +REF-AUR-001,GLFDIIKKIAESF,aurein_1,200,SELECTIVE,Rozek_2000 +REF-AUR-002,GLFDIVKKVVGALGSL,aurein_2,100,SELECTIVE,Rozek_2000 +REF-BUF-001,TRSSRAGLQFPVGRVHRLLRK,buforin_2,200,SELECTIVE,Kim_1996_Eur_J_Biochem +REF-BUF-002,RAGLQFPVGRVHRLLRK,buforin_2_fragment,200,SELECTIVE,Park_2000 +REF-PRG1-001,RGGRLCYCRRRFCVCVGR,protegrin_1,5,HEMOLYTIC,Kokryakov_1993 +REF-PRG2-001,RGGRLCYCRRRFCICVGR,protegrin_2,5,HEMOLYTIC,Kokryakov_1993 +REF-PRG3-001,RGGGLCYCRRRFCVCVGR,protegrin_3,10,MODERATE,Kokryakov_1993 +REF-TACH1-001,KWCFRVCYRGICYRRCR,tachyplesin_I,10,MODERATE,Nakamura_1988 +REF-TACH2-001,RWCFRVCYRGICYRKCR,tachyplesin_II,10,MODERATE,Miyata_1989 +REF-KWK-001,KWKLFKKIGAVLKVL,magainin_template,50,MODERATE,Oren_1997 +REF-RRW-001,RRWQWRMKKLG,tachyplesin_fragment,100,SELECTIVE,Tam_2002 +REF-BMA-001,GRFKRFRKKFKKLFKKLSP,bmap_fragment,10,MODERATE,Skerlavaj_1999 +REF-FKK-001,FKKLKKSLKL,klaklak,100,SELECTIVE,Javadpour_1996 +REF-RLK-001,RLKKVWKKWKK,cationic_trp,50,MODERATE,Parisien_2008 +REF-GRM-001,GRMKKVFFKSLRQH,gramicidin_S_like,1,HEMOLYTIC,Fontenot_1993 +REF-PLA-001,KKVVFKVKFK,short_cationic,100,SELECTIVE,Lee_2004 +REF-GKK-001,GKKLFSKLKK,cecropin_melittin_hybrid,100,SELECTIVE,Gazit_1994 +REF-HIS5-001,DSHAKRHHGYKRKFHEKHHSHRGY,histatin_5,200,SELECTIVE,Oppenheim_1988 +REF-API1-001,GNNRPVYIPQPRPPHPRI,apidaecin_Ia,200,SELECTIVE,Casteels_1989 +REF-DROS-001,GKPRPYSPRPTSHPRPIRV,drosocin,200,SELECTIVE,Bulet_1993 +REF-DRS1-001,ALWKTMLKKLGTMALHAGKAALGAAADTISQGTQ,dermaseptin_S1,25,MODERATE,Mor_1991 +REF-DRS3-001,ALWKNMLKGIGKLAGQAALGAVKTLVGAETQ,dermaseptin_S3,50,MODERATE,Mor_1994 +REF-PLUR-001,GWGSFFKKAAHVGKHVGKAALTHYL,pleurocidin,50,MODERATE,Cole_1997 +REF-PISC1-001,FFHHIFRGIVHVGKTIHLRLTG,piscidin_1,5,HEMOLYTIC,Silphaduang_2001 +REF-PISC3-001,FFHHIFRGLIHVGLTIHLRITG,piscidin_3,10,MODERATE,Silphaduang_2001 +REF-AREN1-001,RWCVYAYVRVRGVLVRYRRCW,arenicin_1,10,MODERATE,Ovchinnikova_2004 +REF-ESC-001,GIFSKLAGKKIKNLLISGLKG,esculentin,100,SELECTIVE,Simmaco_1993 +REF-BOM-001,IIGPVLGLVGSALGGLLKKI,bombinin_H,100,SELECTIVE,Halverson_1990 +REF-TMPG-001,ILSKIFSLLKKVLP,temporin_G,100,SELECTIVE,Simmaco_1996 +REF-POLY1-001,RRWCFRVCYRGFCYRKCR,polyphemusin_I,10,MODERATE,Miyata_1989 diff --git a/src/openamp_foundry/benchmark/retrospective.py b/src/openamp_foundry/benchmark/retrospective.py index ff3df73f..321a1211 100644 --- a/src/openamp_foundry/benchmark/retrospective.py +++ b/src/openamp_foundry/benchmark/retrospective.py @@ -756,3 +756,284 @@ def run_expert_ablation_benchmark( "Wet-lab validation remains mandatory." ), } + + +# ───────────────────────────────────────────────────────────────────────────── +# Within-AMP selectivity benchmark +# ───────────────────────────────────────────────────────────────────────────── + +def run_selectivity_benchmark( + hemolysis_csv: str | Path, + config_path: str | Path = "configs/pipeline.yaml", + n_bootstrap: int = 2000, +) -> dict: + """Within-AMP selectivity benchmark: can scorers distinguish hemolytic from selective AMPs? + + The standard and expert-ablation benchmarks measure AMP-vs-decoy discrimination: + can the pipeline tell AMPs from random peptides? The expert ablation found that + safety, synthesis, and serum stability are anti-signal on that task — real AMPs + have extreme biophysical properties that safety/stability scorers penalise. + + But those scorers were not designed for AMP-vs-decoy discrimination. They were + designed for *within-AMP ranking*: given two peptides that are both AMP-like, + which one is more likely to spare mammalian cells? This benchmark tests that. + + Design: + - Positives (hemolytic): AMPs with literature HC50 < 25 µg/mL (HEMOLYTIC + MODERATE + classes with HC50 ≤ 25). + - Negatives (selective): AMPs with literature HC50 ≥ 100 µg/mL (SELECTIVE class). + - MODERATE peptides with HC50 25–100 are excluded from the binary task to create + a cleaner separation, but are reported separately as a "border zone" diagnostic. + - The benchmark computes AUROC for every pipeline score (ensemble, activity, + safety, selectivity_proxy, synthesis, serum_stability, boman_activity) and + the expert composite, asking: which scores rank hemolytic AMPs *higher* than + selective AMPs? + + A score that ranks hemolytic AMPs higher has AUROC > 0.5 (it detects hemolysis + risk). A score that ranks selective AMPs higher has AUROC < 0.5 (it detects + selectivity). The sign matters: for a safety scorer, AUROC < 0.5 is correct + (safety should be lower for hemolytic peptides). We report both the raw AUROC + and a "hemolysis-detection AUROC" where low-safety → high-risk is the correct + direction. + + Args: + hemolysis_csv: CSV with columns id, sequence, family, hc50_ugml, + hemolysis_class, reference. + config_path: Pipeline config for scoring weights. + n_bootstrap: Bootstrap resample count for CIs. + + Returns: + Dict with per-score AUROC, bootstrap CIs, hemolytic vs selective score + distributions, border-zone diagnostics, and an honest verdict. + """ + from openamp_foundry.config import load_config + from openamp_foundry.features.physchem import compute_features + from openamp_foundry.scoring.activity import activity_likeness_score + from openamp_foundry.scoring.boman import boman_activity_score, model_disagreement + from openamp_foundry.scoring.ensemble import ensemble_score + from openamp_foundry.scoring.expert import expert_score + from openamp_foundry.scoring.novelty import novelty_score + from openamp_foundry.scoring.safety import safety_score + from openamp_foundry.scoring.stability import serum_stability_score + from openamp_foundry.scoring.synthesis import synthesis_feasibility_score + + config = load_config(config_path) + weights = config["weights"] + + rows: list[dict] = [] + with open(hemolysis_csv, newline="", encoding="utf-8") as f: + for row in csv.DictReader(f): + seq = row["sequence"].strip().upper() + if not all(aa in "ACDEFGHIKLMNPQRSTVWY" for aa in seq): + continue + hc50 = float(row["hc50_ugml"]) + hemo_class = row["hemolysis_class"].strip().upper() + features = compute_features(seq) + act = activity_likeness_score(features) + safe = safety_score(features) + synth = synthesis_feasibility_score(features, valid_sequence=True) + nov, _ = novelty_score(seq, []) + boman_act = boman_activity_score(seq) + stability = serum_stability_score(features) + sel_proxy = float(features.get("selectivity_proxy", 0.0)) + raw = { + "activity": act, "safety": safe, + "synthesis": synth, "novelty": nov, + "boman_activity": boman_act, + "disagreement": model_disagreement(act, boman_act), + "serum_stability": stability, + "selectivity_proxy": sel_proxy, + } + ens = ensemble_score(raw, weights) + exp = expert_score(seq, features=features) + + rows.append({ + "id": row["id"], + "sequence": seq, + "family": row.get("family", ""), + "hc50": hc50, + "hemolysis_class": hemo_class, + "ensemble": ens, + "expert_composite": exp.composite, + "activity": act, + "safety": safe, + "synthesis": synth, + "novelty": nov, + "boman_activity": boman_act, + "serum_stability": stability, + "selectivity_proxy": sel_proxy, + "hinge_selectivity": exp.components["hinge_selectivity"], + }) + + # Binary task: hemolytic (HC50 < 25) vs selective (HC50 >= 100) + hemolytic = [r for r in rows if r["hc50"] < 25] + selective = [r for r in rows if r["hc50"] >= 100] + border = [r for r in rows if 25 <= r["hc50"] < 100] + + n_hemo = len(hemolytic) + n_sel = len(selective) + n_border = len(border) + + # Score columns to evaluate + score_cols = [ + "ensemble", "expert_composite", "activity", "safety", + "synthesis", "novelty", "boman_activity", "serum_stability", + "selectivity_proxy", "hinge_selectivity", + ] + + per_score: dict[str, dict] = {} + for col in score_cols: + hemo_scores = [r[col] for r in hemolytic] + sel_scores = [r[col] for r in selective] + auroc = round(_auc_wilcoxon(hemo_scores, sel_scores), 4) + ci = _bootstrap_auroc_ci(hemo_scores, sel_scores, n_bootstrap=n_bootstrap) + + # For safety/selectivity scorers, the CORRECT direction is: + # hemolytic AMPs should score LOWER (less safe, less selective). + # So the "hemolysis-detection AUROC" = 1 - raw AUROC for those axes. + # We report both and let the verdict interpret. + detection_auroc = round(1.0 - auroc, 4) + detection_ci = { + "ci_lo": round(1.0 - ci["ci_hi"], 4), + "ci_hi": round(1.0 - ci["ci_lo"], 4), + } + + per_score[col] = { + "auroc": auroc, + "ci95_lo": ci["ci_lo"], + "ci95_hi": ci["ci_hi"], + # hemolysis_detection_auroc: AUROC when the positive class is "hemolytic" + # and higher score = higher risk. For safety, detection_auroc = 1 - auroc + # because lower safety = higher risk. + "hemolysis_detection_auroc": detection_auroc, + "detection_ci95_lo": detection_ci["ci_lo"], + "detection_ci95_hi": detection_ci["ci_hi"], + "mean_hemolytic": round(sum(hemo_scores) / n_hemo, 4) if hemo_scores else None, + "mean_selective": round(sum(sel_scores) / n_sel, 4) if sel_scores else None, + "direction": "higher_is_risk" if auroc > 0.5 else "higher_is_safe", + } + + # Identify which scores detect hemolysis risk (detection AUROC > 0.5 with CI lo > 0.5) + risk_detectors = [ + col for col, info in per_score.items() + if info["hemolysis_detection_auroc"] > 0.5 + and info["detection_ci95_lo"] > 0.5 + ] + risk_indicators = [ + col for col, info in per_score.items() + if info["hemolysis_detection_auroc"] > 0.5 + and info["detection_ci95_lo"] <= 0.5 + ] + + # Key question: does the safety scorer detect hemolysis? + safety_detection = per_score["safety"]["hemolysis_detection_auroc"] + safety_ci_lo = per_score["safety"]["detection_ci95_lo"] + + if safety_detection > 0.5 and safety_ci_lo > 0.5: + safety_verdict = ( + f"Safety scorer detects hemolysis risk (detection AUROC={safety_detection:.4f}, " + f"CI lo={safety_ci_lo:.4f}). This is the first evidence that the safety scorer " + "earns its keep on the within-AMP selectivity task." + ) + elif safety_detection > 0.5: + safety_verdict = ( + f"Safety scorer shows weak hemolysis detection (detection AUROC={safety_detection:.4f}, " + f"CI lo={safety_ci_lo:.4f} < 0.5). Signal is present but not statistically significant. " + "The melittin blind spot remains: 1D hydrophobic moment cannot capture all hemolysis mechanisms." + ) + else: + safety_verdict = ( + f"Safety scorer FAILS to detect hemolysis risk (detection AUROC={safety_detection:.4f}). " + "This confirms the expert ablation finding: the safety scorer does not distinguish " + "hemolytic from selective AMPs. Hemolysis must be assayed experimentally." + ) + + # Selectivity proxy assessment + sel_detection = per_score["selectivity_proxy"]["hemolysis_detection_auroc"] + sel_ci_lo = per_score["selectivity_proxy"]["detection_ci95_lo"] + + if sel_detection > 0.5 and sel_ci_lo > 0.5: + sel_verdict = ( + f"Selectivity proxy detects hemolysis risk (detection AUROC={sel_detection:.4f}, " + f"CI lo={sel_ci_lo:.4f}). The charge/GRAVY heuristic captures the selectivity signal." + ) + elif sel_detection > 0.5: + sel_verdict = ( + f"Selectivity proxy shows weak hemolysis detection (detection AUROC={sel_detection:.4f}, " + f"CI lo={sel_ci_lo:.4f}). Trend is correct but not significant at this sample size." + ) + else: + sel_verdict = ( + f"Selectivity proxy does NOT detect hemolysis risk (detection AUROC={sel_detection:.4f}). " + "The charge/GRAVY heuristic does not capture hemolysis on this reference set." + ) + + # Expert composite assessment + exp_detection = per_score["expert_composite"]["hemolysis_detection_auroc"] + ens_detection = per_score["ensemble"]["hemolysis_detection_auroc"] + + if exp_detection > ens_detection + 0.02: + expert_verdict = ( + f"Expert composite detects hemolysis risk better than ensemble " + f"(detection AUROC {exp_detection:.4f} vs {ens_detection:.4f}). " + "The added selectivity/safety components earn their keep on within-AMP ranking." + ) + elif abs(exp_detection - ens_detection) <= 0.02: + expert_verdict = ( + f"Expert composite ({exp_detection:.4f}) and ensemble ({ens_detection:.4f}) " + "are within ±0.02 on hemolysis detection. The expert components do not materially " + "improve within-AMP selectivity on this reference set." + ) + else: + expert_verdict = ( + f"Expert composite detects hemolysis risk worse than ensemble " + f"(detection AUROC {exp_detection:.4f} vs {ens_detection:.4f}). " + "The expert components degrade within-AMP selectivity on this reference set." + ) + + # Rank hemolytic AMPs by safety score to find blind spots + hemolytic_by_safety = sorted(hemolytic, key=lambda r: r["safety"], reverse=True) + blind_spots = [ + {"id": r["id"], "sequence": r["sequence"], "family": r["family"], + "hc50": r["hc50"], "safety": r["safety"], "selectivity_proxy": r["selectivity_proxy"]} + for r in hemolytic_by_safety if r["safety"] >= 0.8 + ] + + return { + "benchmark": "within_amp_selectivity", + "n_total": len(rows), + "n_hemolytic": n_hemo, + "n_selective": n_sel, + "n_border": n_border, + "hemolytic_threshold_hc50": 25, + "selective_threshold_hc50": 100, + "per_score_auroc": per_score, + "risk_detectors": risk_detectors, + "risk_indicators": risk_indicators, + "safety_verdict": safety_verdict, + "selectivity_proxy_verdict": sel_verdict, + "expert_composite_verdict": expert_verdict, + "blind_spots": blind_spots, + "border_zone": [ + {"id": r["id"], "sequence": r["sequence"], "family": r["family"], + "hc50": r["hc50"], "safety": r["safety"], + "selectivity_proxy": r["selectivity_proxy"]} + for r in sorted(border, key=lambda r: r["hc50"]) + ], + "design_note": ( + "Within-AMP selectivity benchmark: all sequences are confirmed AMPs. " + "The task is NOT 'is this an AMP?' but 'is this AMP likely to be hemolytic?'. " + "Hemolytic class: HC50 < 25 µg/mL. Selective class: HC50 >= 100 µg/mL. " + "Border zone (25-100) excluded from binary AUROC, reported separately. " + "HC50 values are approximate literature values from multiple sources; " + "they vary with assay conditions (RBC source, buffer, incubation time). " + "This is a coarse triage benchmark, not a calibrated hemolysis predictor." + ), + "disclaimer": ( + "HC50 values are approximate literature values with high inter-assay variability. " + "This benchmark tests whether the pipeline's physicochemical scorers can distinguish " + "hemolytic from selective AMPs. It does NOT predict any individual candidate's " + "hemolysis. Hemolysis must be assayed experimentally for every candidate. " + "Wet-lab validation remains mandatory." + ), + } diff --git a/src/openamp_foundry/cli/commands/benchmark.py b/src/openamp_foundry/cli/commands/benchmark.py index 85a0efbf..7721eefd 100644 --- a/src/openamp_foundry/cli/commands/benchmark.py +++ b/src/openamp_foundry/cli/commands/benchmark.py @@ -151,3 +151,34 @@ def _run_expert_ablation_bench(args: argparse.Namespace) -> int: } print(_json.dumps(summary, indent=2)) return 0 + + +def _run_selectivity_bench(args: argparse.Namespace) -> int: + """Run within-AMP selectivity benchmark (hemolytic vs selective AMPs).""" + import json as _json + from openamp_foundry.benchmark.retrospective import run_selectivity_benchmark + from openamp_foundry.utils.io import write_json + + result = run_selectivity_benchmark( + hemolysis_csv=args.hemolysis_csv, + config_path=args.config, + n_bootstrap=args.n_bootstrap, + ) + if args.out: + write_json(args.out, result) + summary = { + "status": "ok", + "benchmark": "within_amp_selectivity", + "n_hemolytic": result["n_hemolytic"], + "n_selective": result["n_selective"], + "n_border": result["n_border"], + "risk_detectors": result["risk_detectors"], + "risk_indicators": result["risk_indicators"], + "safety_verdict": result["safety_verdict"], + "selectivity_proxy_verdict": result["selectivity_proxy_verdict"], + "expert_composite_verdict": result["expert_composite_verdict"], + "n_blind_spots": len(result["blind_spots"]), + "out": args.out, + } + print(_json.dumps(summary, indent=2)) + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 79c4f0e1..c80cac2c 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -1,7 +1,7 @@ from __future__ import annotations from openamp_foundry.cli.commands.core import _run_generate_batch -from openamp_foundry.cli.commands.benchmark import _run_bench, _run_validate_scoring, _run_cluster_split_bench, _run_expert_ablation_bench +from openamp_foundry.cli.commands.benchmark import _run_bench, _run_validate_scoring, _run_cluster_split_bench, _run_expert_ablation_bench, _run_selectivity_bench from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc @@ -140,6 +140,36 @@ def build_parser() -> argparse.ArgumentParser: help="Optional JSON output path.", ) + selectivity = bench_sub.add_parser( + "selectivity", + help=( + "Within-AMP selectivity benchmark: can pipeline scorers distinguish " + "hemolytic AMPs (HC50 < 25 ug/mL) from selective AMPs (HC50 >= 100 ug/mL)? " + "Tests whether the safety, selectivity, and synthesis components earn " + "their keep on within-AMP ranking -- the task they were designed for." + ), + ) + selectivity.add_argument( + "--hemolysis-csv", + default="examples/validation/hemolysis_reference.csv", + help="CSV with columns: id, sequence, family, hc50_ugml, hemolysis_class, reference.", + ) + selectivity.add_argument( + "--config", + default="configs/pipeline.yaml", + ) + selectivity.add_argument( + "--n-bootstrap", + type=int, + default=2000, + help="Bootstrap resample count (default: 2000).", + ) + selectivity.add_argument( + "--out", + required=False, + help="Optional JSON output path.", + ) + generate = sub.add_parser( "generate-batch", help=( @@ -587,6 +617,8 @@ def main(argv: list[str] | None = None) -> int: return _run_cluster_split_bench(args) if args.bench_command == "expert-ablation": return _run_expert_ablation_bench(args) + if args.bench_command == "selectivity": + return _run_selectivity_bench(args) return _run_bench(args) if args.command == "generate-batch": diff --git a/tests/test_expert_ablation.py b/tests/test_expert_ablation.py index 2e38891a..376ac506 100644 --- a/tests/test_expert_ablation.py +++ b/tests/test_expert_ablation.py @@ -15,7 +15,6 @@ """ from __future__ import annotations -import csv import pytest from openamp_foundry.benchmark.retrospective import run_expert_ablation_benchmark diff --git a/tests/test_selectivity_benchmark.py b/tests/test_selectivity_benchmark.py new file mode 100644 index 00000000..2183ec60 --- /dev/null +++ b/tests/test_selectivity_benchmark.py @@ -0,0 +1,189 @@ +"""Within-AMP selectivity benchmark tests. + +Tests the benchmark that asks: can the pipeline's physicochemical scorers +distinguish hemolytic AMPs (HC50 < 25 µg/mL) from selective AMPs (HC50 >= 100 µg/mL)? + +This is the complementary benchmark to the expert ablation: the ablation tested +AMP-vs-decoy discrimination (where safety/synthesis were anti-signal); this tests +within-AMP selectivity, the task those scorers were designed for. + +All results are computational. No biological activity is implied. +""" +from __future__ import annotations + +import csv +import tempfile + +import pytest + +from openamp_foundry.benchmark.retrospective import run_selectivity_benchmark + + +HEMOLYSIS_CSV = "examples/validation/hemolysis_reference.csv" + +SCORE_COLS = [ + "ensemble", "expert_composite", "activity", "safety", + "synthesis", "novelty", "boman_activity", "serum_stability", + "selectivity_proxy", "hinge_selectivity", +] + + +class TestSelectivityBenchmark: + @pytest.fixture(scope="class") + def result(self): + return run_selectivity_benchmark( + HEMOLYSIS_CSV, n_bootstrap=200, + ) + + def test_benchmark_runs_on_real_data(self, result): + assert result["benchmark"] == "within_amp_selectivity" + assert result["n_hemolytic"] > 0 + assert result["n_selective"] > 0 + assert result["n_total"] == result["n_hemolytic"] + result["n_selective"] + result["n_border"] + + def test_per_score_auroc_has_all_columns(self, result): + per_score = result["per_score_auroc"] + for col in SCORE_COLS: + assert col in per_score + assert 0.0 <= per_score[col]["auroc"] <= 1.0 + assert 0.0 <= per_score[col]["hemolysis_detection_auroc"] <= 1.0 + + def test_detection_auroc_is_complement(self, result): + """Detection AUROC = 1 - AUROC for every score.""" + for col, info in result["per_score_auroc"].items(): + assert info["hemolysis_detection_auroc"] == pytest.approx( + 1.0 - info["auroc"], abs=1e-4 + ) + + def test_ci_bounds_are_ordered(self, result): + for col, info in result["per_score_auroc"].items(): + assert info["ci95_lo"] <= info["ci95_hi"] + assert info["detection_ci95_lo"] <= info["detection_ci95_hi"] + + def test_mean_scores_are_present(self, result): + for col, info in result["per_score_auroc"].items(): + assert info["mean_hemolytic"] is not None + assert info["mean_selective"] is not None + + def test_safety_verdict_present(self, result): + assert "safety" in result["safety_verdict"].lower() + assert "hemolysis" in result["safety_verdict"].lower() + + def test_selectivity_proxy_verdict_present(self, result): + assert "selectivity" in result["selectivity_proxy_verdict"].lower() + + def test_expert_composite_verdict_present(self, result): + assert "expert" in result["expert_composite_verdict"].lower() + assert "ensemble" in result["expert_composite_verdict"].lower() + + def test_blind_spots_listed(self, result): + """All hemolytic AMPs with safety >= 0.8 should be listed as blind spots.""" + blind = result["blind_spots"] + for bp in blind: + assert bp["safety"] >= 0.8 + assert bp["hc50"] < 25 + + def test_border_zone_present(self, result): + """Border zone peptides (25 <= HC50 < 100) should be reported.""" + border = result["border_zone"] + for bp in border: + assert 25 <= bp["hc50"] < 100 + + def test_disclaimer_present(self, result): + assert "wet-lab" in result["disclaimer"].lower() + assert "physicochemical" in result["disclaimer"].lower() + + def test_design_note_present(self, result): + assert "within-amp" in result["design_note"].lower() + assert "hemolytic" in result["design_note"].lower() + + def test_risk_detectors_and_indicators_partitioned(self, result): + """Risk detectors (CI lo > 0.5) and indicators (point > 0.5, CI lo <= 0.5) are disjoint.""" + detectors = set(result["risk_detectors"]) + indicators = set(result["risk_indicators"]) + assert detectors & indicators == set() + + def test_safety_scorer_is_not_a_risk_detector(self, result): + """The safety scorer should NOT be a significant hemolysis risk detector. + + This confirms the known melittin blind spot: the 1D hydrophobic moment + cannot capture all hemolysis mechanisms. If this test fails, the safety + scorer has improved and the blind spot may be closing. + """ + assert "safety" not in result["risk_detectors"] + + +class TestSelectivityBenchmarkEdgeCases: + """Test edge cases with small synthetic datasets.""" + + def _write_csv(self, rows: list[dict]) -> str: + f = tempfile.NamedTemporaryFile( + mode="w", suffix=".csv", delete=False, newline="" + ) + writer = csv.DictWriter( + f, + fieldnames=["id", "sequence", "family", "hc50_ugml", "hemolysis_class", "reference"], + ) + writer.writeheader() + for row in rows: + writer.writerow(row) + f.close() + return f.name + + def test_all_hemolytic(self): + """If all peptides are hemolytic, AUROC should be 0.5 (no negatives).""" + rows = [ + {"id": "H1", "sequence": "GIGAVLKVLTTGLPALISWIKRKRQQ", "family": "melittin", + "hc50_ugml": "1.5", "hemolysis_class": "HEMOLYTIC", "reference": "test"}, + {"id": "H2", "sequence": "RGGRLCYCRRRFCVCVGR", "family": "protegrin", + "hc50_ugml": "5", "hemolysis_class": "HEMOLYTIC", "reference": "test"}, + ] + csv_path = self._write_csv(rows) + result = run_selectivity_benchmark(csv_path, n_bootstrap=50) + assert result["n_selective"] == 0 + # With no selective class, AUROC should be 0.5 + for col, info in result["per_score_auroc"].items(): + assert info["auroc"] == 0.5 + + def test_all_selective(self): + """If all peptides are selective, AUROC should be 0.5 (no positives).""" + rows = [ + {"id": "S1", "sequence": "GIGKFLHSAKKFGKAFVGEIMNS", "family": "magainin", + "hc50_ugml": "100", "hemolysis_class": "SELECTIVE", "reference": "test"}, + {"id": "S2", "sequence": "DSHAKRHHGYKRKFHEKHHSHRGY", "family": "histatin", + "hc50_ugml": "200", "hemolysis_class": "SELECTIVE", "reference": "test"}, + ] + csv_path = self._write_csv(rows) + result = run_selectivity_benchmark(csv_path, n_bootstrap=50) + assert result["n_hemolytic"] == 0 + + def test_nonstandard_aa_skipped(self): + """Sequences with non-standard amino acids should be silently skipped.""" + rows = [ + {"id": "H1", "sequence": "GIGAVLKVLTTGLPALISWIKRKRQQ", "family": "melittin", + "hc50_ugml": "1.5", "hemolysis_class": "HEMOLYTIC", "reference": "test"}, + {"id": "X1", "sequence": "XBZBQ", "family": "nonstandard", + "hc50_ugml": "1", "hemolysis_class": "HEMOLYTIC", "reference": "test"}, + {"id": "S1", "sequence": "GIGKFLHSAKKFGKAFVGEIMNS", "family": "magainin", + "hc50_ugml": "100", "hemolysis_class": "SELECTIVE", "reference": "test"}, + ] + csv_path = self._write_csv(rows) + result = run_selectivity_benchmark(csv_path, n_bootstrap=50) + assert result["n_total"] == 2 # only H1 and S1 + assert result["n_hemolytic"] == 1 + assert result["n_selective"] == 1 + + def test_perfect_separation(self): + """If hemolytic and selective peptides score perfectly apart, AUROC = 1.0 or 0.0.""" + # Use peptides where activity scorer will give very different scores + rows = [ + {"id": "H1", "sequence": "GIGAVLKVLTTGLPALISWIKRKRQQ", "family": "melittin", + "hc50_ugml": "1.5", "hemolysis_class": "HEMOLYTIC", "reference": "test"}, + {"id": "S1", "sequence": "DSHAKRHHGYKRKFHEKHHSHRGY", "family": "histatin", + "hc50_ugml": "200", "hemolysis_class": "SELECTIVE", "reference": "test"}, + ] + csv_path = self._write_csv(rows) + result = run_selectivity_benchmark(csv_path, n_bootstrap=50) + # With 1 vs 1, AUROC is either 0, 0.5, or 1 + for col, info in result["per_score_auroc"].items(): + assert info["auroc"] in (0.0, 0.5, 1.0)