diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 45304031..5f1b9fa1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -37,9 +37,9 @@ The purpose of the added layers is not simulation theater. It is to improve whic | `openamp_foundry.data` | loading and normalizing candidate/reference data | | `openamp_foundry.features` | physicochemical feature extraction | | `openamp_foundry.scoring` | activity, safety, hemolysis risk, novelty, synthesis, ensemble, expert composite scoring | -| `openamp_foundry.selection` | ranking and diversity selection | +| `openamp_foundry.selection` | ranking (ensemble or expert composite) and diversity selection | | `openamp_foundry.evidence` | JSON certificate generation and validation | -| `openamp_foundry.benchmark` | leakage checks, cluster-split benchmark, expert ablation benchmark, within-AMP selectivity benchmark, and evaluation scaffolding | +| `openamp_foundry.benchmark` | leakage checks, cluster-split benchmark, expert ablation benchmark, within-AMP selectivity benchmark, triage benchmark, and evaluation scaffolding | | `openamp_foundry.generators` | safe, bounded toy candidate generation | | `openamp_foundry.simulation` | membrane/selectivity/stability proxy modeling (scaffolded) | diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index 06906174..a8d25a49 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -475,4 +475,5 @@ The MODERATE class (HC50 25-100, n=68) is excluded from the binary task. | 2026-07-01 | **Hemolysis benchmark expanded:** 42 -> 238 peptides using DBAASP human erythrocyte data (54 hemolytic vs 125 selective, n=179 binary). Hemolysis risk scorer detection AUROC drops 0.9218 -> 0.5650 (CI 0.47-0.66) — original performance was small-sample inflation. Direction correct, not significant. Safety scorer detection improves 0.3844 -> 0.5116 (still not significant). 196 new peptides from DBAASP v3. | OpenAMP loop | | 2026-07-01 | Dedicated hemolysis risk scorer: 4-component score (synth+aromatic+face+cys) achieves detection AUROC=0.9218 (CI: 0.82-0.99); integrated into expert composite (detection 0.5119→0.6429); safety scorer unchanged; 1471 tests | OpenAMP loop | | 2026-07-01 | Within-AMP selectivity benchmark added: safety scorer FAILS hemolysis detection (AUROC=0.3844); synthesis is only significant risk detector (AUROC=0.8027); expert composite better than ensemble but not significant (0.5119 vs 0.3486) | OpenAMP loop | +| 2026-07-01 | Expert composite ranking integration: `score_candidates()` now computes `expert_composite` and `hemolysis_risk`; `--ranking-mode expert` CLI flag; expert-ranked top-5 have lower mean hemolysis_risk than ensemble | OpenAMP loop | | 2026-06-29 | Initial — expanded benchmark (PR #110) | OpenAMP CI | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 01a1bb58..13c696ca 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -209,6 +209,32 @@ Implemented during the pre-wet-lab improvement loop (PRs #31–#54): - Makefile: `make bench-triage` - 16 new tests (15 triage benchmark + 1 CLI integration); 1487 tests total. +## v0.5.13 — Expert composite ranking integration ✓ (2026-07-01) + +The expert composite scorer (`scoring/expert.py`) was benchmarked but never connected +to the pipeline. This integration makes it available as an optional ranking mode. + +Changes: +- `score_candidates()` now computes `expert_composite` and `hemolysis_risk` for every + candidate (stored in `scores` dict alongside `ensemble`) +- `rank_candidates()` accepts `ranking_mode="expert"` to rank by expert composite + instead of ensemble (default remains "ensemble") +- `run_ranking_pipeline()` accepts `ranking_mode` parameter, recorded in batch report +- CLI: `openamp-foundry rank --ranking-mode expert` flag added +- `selection/pareto.py`: `select_top()` also accepts `ranking_mode` + +Why this matters: +The triage benchmark (v0.5.12) showed the ensemble has an anti-selective bias +(sel_vs_hem AUROC=0.466). The expert composite incorporates selectivity_proxy, +hemolysis risk, helix-hinge analysis, and motif novelty — partially correcting +this bias. Expert-ranked top-5 candidates have lower mean hemolysis_risk than +ensemble-ranked top-5 on the mixed benchmark set (verified by test). + +Honest limitation: The expert composite's hemolysis components are not statistically +significant detectors on the expanded n=179 benchmark (AUROC=0.565, CI 0.47-0.66). +Expert ranking is a safety-aware alternative, not a validated safety guarantee. +Wet-lab hemolysis assay remains mandatory. + ## v1.0 — Validated dry-lab-to-wet-lab loop - independently reviewed assay batch (expert_review.yml GitHub issue template); diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index baa7ecb7..9a6056db 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -29,6 +29,17 @@ def build_parser() -> argparse.ArgumentParser: rank.add_argument("--cert-dir", required=False) rank.add_argument("--manifest", required=False) rank.add_argument("--config", default="configs/pipeline.yaml") + rank.add_argument( + "--ranking-mode", + choices=["ensemble", "expert"], + default="ensemble", + help=( + "Ranking mode: 'ensemble' (default, activity-weighted) or " + "'expert' (safety-aware expert composite with selectivity, " + "hemolysis risk, and helix-hinge analysis). Expert mode " + "partially corrects the ensemble's anti-selective bias." + ), + ) validate = sub.add_parser("validate", help="Validate a candidate certificate against JSON schema.") validate.add_argument("--certificate", required=True) @@ -628,6 +639,7 @@ def main(argv: list[str] | None = None) -> int: cert_dir=args.cert_dir, config_path=args.config, manifest_path=args.manifest, + ranking_mode=getattr(args, "ranking_mode", "ensemble"), ) print(json.dumps({"status": "ok", "out": args.out, "report": args.report}, indent=2)) return 0 diff --git a/src/openamp_foundry/pipeline.py b/src/openamp_foundry/pipeline.py index 1108a99b..6a8f7920 100644 --- a/src/openamp_foundry/pipeline.py +++ b/src/openamp_foundry/pipeline.py @@ -17,6 +17,7 @@ from openamp_foundry.scoring.safety import safety_score from openamp_foundry.scoring.stability import serum_stability_score from openamp_foundry.scoring.synthesis import synthesis_feasibility_score +from openamp_foundry.scoring.hemolysis import hemolysis_risk_score from openamp_foundry.selection.diversity import greedy_diverse_select from openamp_foundry.selection.pareto import rank_candidates from openamp_foundry.types import ScoredCandidate @@ -67,8 +68,16 @@ def score_candidates( # mammalian cytotoxicity. Computed from net charge and GRAVY in compute_features(). # Stored in scores (not just features) so pilot_priority can use it for ranking. "selectivity_proxy": features.get("selectivity_proxy", 1.0), + "hemolysis_risk": hemolysis_risk_score(features) if valid else 1.0, } raw_scores["ensemble"] = ensemble_score(raw_scores, weights) + # Expert composite: safety-aware multi-component score that addresses + # the ensemble's anti-selective bias (triage benchmark: ensemble + # ranks hemolytic > selective; expert composite partially corrects). + # Not used for ranking by default — enabled via --ranking-mode expert. + from openamp_foundry.scoring.expert import expert_score + exp = expert_score(candidate.sequence, features=features) + raw_scores["expert_composite"] = exp.composite item = ScoredCandidate( candidate=candidate, features=features, @@ -122,6 +131,7 @@ def run_ranking_pipeline( cert_dir: str | Path | None = None, config_path: str | Path = "configs/pipeline.yaml", manifest_path: str | Path | None = None, + ranking_mode: str = "ensemble", ) -> list[ScoredCandidate]: run_id = str(uuid.uuid4()) generated_at = datetime.now(timezone.utc).isoformat() @@ -132,7 +142,7 @@ def run_ranking_pipeline( max_safety_risk = float(selection_cfg.get("max_safety_risk", 1.0)) max_disagreement = float(selection_cfg.get("max_disagreement", 1.0)) - ranked = rank_candidates(scored) + ranked = rank_candidates(scored, ranking_mode=ranking_mode) top_n = int(selection_cfg.get("top_n", len(ranked))) eligible = [ @@ -171,7 +181,7 @@ def run_ranking_pipeline( if report_path: write_report(report_path, ranked, selected) - batch_report = build_batch_report(ranked, selected, generated_at) + batch_report = build_batch_report(ranked, selected, generated_at, ranking_mode=ranking_mode) report_json = Path(report_path).with_suffix(".json") write_json(report_json, batch_report) @@ -204,6 +214,7 @@ def build_batch_report( ranked: list[ScoredCandidate], selected: list[ScoredCandidate], generated_at: str, + ranking_mode: str = "ensemble", ) -> dict[str, Any]: """Build a machine-readable batch report validatable against batch_report.schema.json.""" selected_ids = {item.candidate.candidate_id for item in selected} @@ -232,6 +243,7 @@ def build_batch_report( ) if ranked else 0.0, }, "selected_ids": sorted(selected_ids), + "ranking_mode": ranking_mode, } diff --git a/src/openamp_foundry/selection/pareto.py b/src/openamp_foundry/selection/pareto.py index 2a9b944a..50f9caf8 100644 --- a/src/openamp_foundry/selection/pareto.py +++ b/src/openamp_foundry/selection/pareto.py @@ -3,9 +3,30 @@ from openamp_foundry.types import ScoredCandidate -def rank_candidates(candidates: list[ScoredCandidate]) -> list[ScoredCandidate]: - return sorted(candidates, key=lambda c: c.scores.get("ensemble", 0.0), reverse=True) +def rank_candidates( + candidates: list[ScoredCandidate], + ranking_mode: str = "ensemble", +) -> list[ScoredCandidate]: + """Rank candidates by the specified scoring mode. + Args: + candidates: List of ScoredCandidate objects. + ranking_mode: "ensemble" (default) or "expert". The expert mode + ranks by the expert composite score, which incorporates + selectivity, hemolysis risk, and helix-hinge analysis — + partially correcting the ensemble's anti-selective bias + (triage benchmark: ensemble ranks hemolytic > selective). -def select_top(candidates: list[ScoredCandidate], top_n: int) -> list[ScoredCandidate]: - return rank_candidates(candidates)[:top_n] + Returns: + Candidates sorted by the chosen score, highest first. + """ + key = "expert_composite" if ranking_mode == "expert" else "ensemble" + return sorted(candidates, key=lambda c: c.scores.get(key, 0.0), reverse=True) + + +def select_top( + candidates: list[ScoredCandidate], + top_n: int, + ranking_mode: str = "ensemble", +) -> list[ScoredCandidate]: + return rank_candidates(candidates, ranking_mode=ranking_mode)[:top_n] diff --git a/tests/test_expert_ranking_integration.py b/tests/test_expert_ranking_integration.py new file mode 100644 index 00000000..7374b398 --- /dev/null +++ b/tests/test_expert_ranking_integration.py @@ -0,0 +1,157 @@ +"""Tests for expert composite ranking integration. + +Verifies that: +1. score_candidates() now computes expert_composite and hemolysis_risk for every candidate. +2. rank_candidates(ranking_mode='expert') produces a different order than ensemble. +3. The expert composite partially corrects the anti-selective bias. +4. The CLI --ranking-mode expert flag works end-to-end. +5. The batch report records which ranking mode was used. + +Context: The triage benchmark (v0.5.12) showed the ensemble has an anti-selective +bias (AUROC selective_vs_hemolytic = 0.466, below random). The expert composite +incorporates selectivity, hemolysis risk, and helix-hinge analysis. This test +verifies the integration is correct, not that the expert composite is biologically +validated (it is not). +""" +from __future__ import annotations + +import json + + +from openamp_foundry.pipeline import score_candidates +from openamp_foundry.selection.pareto import rank_candidates + + +class TestExpertCompositeInPipeline: + """Verify expert_composite is computed during scoring.""" + + def test_expert_composite_present_in_scores(self): + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") + for s in scored: + assert "expert_composite" in s.scores, ( + f"{s.candidate.candidate_id} missing expert_composite score" + ) + assert 0.0 <= s.scores["expert_composite"] <= 1.0 + + def test_hemolysis_risk_present_in_scores(self): + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") + for s in scored: + assert "hemolysis_risk" in s.scores, ( + f"{s.candidate.candidate_id} missing hemolysis_risk score" + ) + assert 0.0 <= s.scores["hemolysis_risk"] <= 1.0 + + def test_expert_composite_differs_from_ensemble(self): + """Expert composite should not be identical to ensemble (different formula).""" + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") + for s in scored: + assert s.scores["expert_composite"] != s.scores["ensemble"], ( + f"{s.candidate.candidate_id}: expert_composite equals ensemble " + "-- they should differ (different component weights)" + ) + + +class TestExpertRankingMode: + """Verify ranking_mode='expert' produces different ordering than ensemble.""" + + def test_expert_ranking_differs_from_ensemble(self): + """At least one position should differ between ensemble and expert ranking.""" + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") + ens_ranked = rank_candidates(scored, ranking_mode="ensemble") + exp_ranked = rank_candidates(scored, ranking_mode="expert") + + ens_ids = [s.candidate.candidate_id for s in ens_ranked] + exp_ids = [s.candidate.candidate_id for s in exp_ranked] + + # With 10 candidates, at least one position should differ + assert ens_ids != exp_ids, ( + "Expert ranking produced identical order to ensemble — " + "the expert composite should change at least some rankings" + ) + + def test_expert_ranking_sorts_by_expert_composite(self): + """Expert ranking should sort by expert_composite, highest first.""" + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") + exp_ranked = rank_candidates(scored, ranking_mode="expert") + + for i in range(len(exp_ranked) - 1): + assert exp_ranked[i].scores["expert_composite"] >= exp_ranked[i + 1].scores["expert_composite"] + + def test_ensemble_ranking_unchanged_by_default(self): + """Default ranking_mode should still be ensemble.""" + scored, _ = score_candidates("examples/sequences/demo_candidates.csv") + default_ranked = rank_candidates(scored) + ens_ranked = rank_candidates(scored, ranking_mode="ensemble") + + default_ids = [s.candidate.candidate_id for s in default_ranked] + ens_ids = [s.candidate.candidate_id for s in ens_ranked] + assert default_ids == ens_ids + + +class TestExpertRankingCLI: + """CLI integration test for --ranking-mode expert.""" + + def test_cli_expert_ranking_produces_valid_output(self, tmp_path): + from openamp_foundry.cli import main + + out = tmp_path / "ranked.jsonl" + report = tmp_path / "report.md" + rc = main([ + "rank", + "--candidates", "examples/sequences/demo_candidates.csv", + "--references", "examples/known_reference/demo_known_amps.csv", + "--out", str(out), + "--report", str(report), + "--ranking-mode", "expert", + ]) + assert rc == 0 + assert out.exists() + + # Batch report should record the ranking mode + report_json = tmp_path / "report.json" + assert report_json.exists() + data = json.loads(report_json.read_text()) + assert data["ranking_mode"] == "expert" + + def test_cli_ensemble_ranking_default(self, tmp_path): + """Default ranking (no --ranking-mode) should be ensemble.""" + from openamp_foundry.cli import main + + out = tmp_path / "ranked.jsonl" + report = tmp_path / "report.md" + rc = main([ + "rank", + "--candidates", "examples/sequences/demo_candidates.csv", + "--references", "examples/known_reference/demo_known_amps.csv", + "--out", str(out), + "--report", str(report), + ]) + assert rc == 0 + report_json = tmp_path / "report.json" + data = json.loads(report_json.read_text()) + assert data["ranking_mode"] == "ensemble" + + +class TestExpertRankingSelectivity: + """Verify expert ranking shifts top candidates toward safer profiles.""" + + def test_expert_top_5_have_lower_hemolysis_risk_on_average(self): + """Expert-ranked top-5 should have lower mean hemolysis_risk than ensemble top-5. + + This is the core value proposition: the expert composite incorporates + hemolysis risk, so top-ranked candidates should be safer on average. + If this test fails, the expert composite's hemolysis components are + not effectively influencing the ranking. + """ + scored, _ = score_candidates("examples/benchmark/mixed_candidates.csv") + ens_ranked = rank_candidates(scored, ranking_mode="ensemble") + exp_ranked = rank_candidates(scored, ranking_mode="expert") + + ens_top5_hemo = sum(s.scores["hemolysis_risk"] for s in ens_ranked[:5]) / 5 + exp_top5_hemo = sum(s.scores["hemolysis_risk"] for s in exp_ranked[:5]) / 5 + + assert exp_top5_hemo <= ens_top5_hemo, ( + f"Expert top-5 mean hemolysis_risk ({exp_top5_hemo:.4f}) should be <= " + f"ensemble top-5 ({ens_top5_hemo:.4f}). The expert composite should " + f"favor safer candidates." + )