diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0919b641..b50d7d3a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -78,6 +78,9 @@ The system is designed to reduce these failures: 8. Write JSONL results. 9. Generate one certificate per selected candidate. 10. Generate a human-readable report. +11. Record the ranking-policy rationale used for the run so reviewers can see + whether the batch used the broad default gate or the narrower safety-aware + alternative. ## Target future data flow diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index 241eafdc..f48aba8d 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -710,6 +710,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | 2026-07-01 | Within-AMP selectivity benchmark added: safety scorer FAILS hemolysis detection (AUROC=0.3844); synthesis is only significant risk detector (AUROC=0.8027); expert composite better than ensemble but not significant (0.5119 vs 0.3486) | OpenAMP loop | | 2026-07-01 | Expert composite ranking integration: `score_candidates()` now computes `expert_composite` and `hemolysis_risk`; `--ranking-mode expert` CLI flag; expert-ranked top-5 have lower mean hemolysis_risk than ensemble | OpenAMP loop | | 2026-07-02 | **Strict triage benchmark added:** composition-matched scrambled decoys replace random background. No scorer triages correctly — standard triage "success" of selectivity_proxy (0.782 sel_vs_dec) and expert_composite (0.757) was inflated by trivially distinguishable decoys. selectivity_proxy collapses to 0.500 (purely composition-driven), ensemble drops to 0.572. Real bottleneck (selective_vs_hemolytic) unchanged. | OpenAMP loop | +| 2026-07-02 | Ranking policy contract added: machine-readable recommendation now states `ensemble` remains default broad synthesis gate, `expert` is narrower safety-aware alternative only | OpenAMP loop | | 2026-07-03 | **Rich selectivity scorer added:** composite of 8 evidence-identified features from the feature decomposition benchmark. Detection AUROC=0.7138 (CI 0.63-0.80) on n=179 — first pipeline score with statistically significant selective_vs_hemolytic discrimination. Old selectivity_proxy=0.5744 (CI 0.50-0.66). Honest limitation: does not triage AMP-vs-decoy (0.19); must be combined with activity gate. | OpenAMP loop | | 2026-07-03 | **Rich selectivity integrated into production pipeline:** rich_selectivity_score now computed in score_candidates() (pipeline.py), replaces hemolysis_safety as the expert composite hemolysis-risk component (weight 0.10), used in pilot_priority formula, displayed in pilot panel report, and included in evidence certificates. Expert AUROC drops 0.7119→0.7097 (−0.0022) — acceptable tradeoff: the expert now includes a significant hemolysis detector (CI excludes 0.5) instead of the old non-significant one. | OpenAMP loop | | 2026-07-03 | **Two-gate triage composite added:** gate_triage = activity × rich_selectivity, added to triage benchmark. First scorer to pass all three standard triage conditions with strong selective_vs_hemolytic separation (0.666). Top-20: 16 selective / 1 hemolytic / 3 decoy — best distribution. Does NOT pass strict triage (hem_vs_dec 0.489) — honest limitation. Must not replace ensemble activity gate. | OpenAMP loop | diff --git a/schemas/batch_report.schema.json b/schemas/batch_report.schema.json index 1ca8eb2d..8c547daf 100644 --- a/schemas/batch_report.schema.json +++ b/schemas/batch_report.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "OpenAMP Batch Report", "type": "object", - "required": ["pipeline_version", "candidate_count", "selected_count", "generated_at", "disclaimer"], + "required": ["pipeline_version", "candidate_count", "selected_count", "generated_at", "disclaimer", "ranking_mode", "ranking_policy"], "properties": { "pipeline_version": {"type": "string"}, "candidate_count": {"type": "integer", "minimum": 0}, @@ -18,6 +18,18 @@ "ensemble": {"type": "number", "minimum": 0, "maximum": 1} } }, - "selected_ids": {"type": "array", "items": {"type": "string"}} + "selected_ids": {"type": "array", "items": {"type": "string"}}, + "ranking_mode": {"type": "string", "enum": ["ensemble", "expert"]}, + "ranking_policy": { + "type": "object", + "required": ["mode", "recommended_for", "evidence_basis", "caution", "not_a_claim"], + "properties": { + "mode": {"type": "string", "enum": ["ensemble", "expert"]}, + "recommended_for": {"type": "string"}, + "evidence_basis": {"type": "array", "items": {"type": "string"}}, + "caution": {"type": "string"}, + "not_a_claim": {"type": "string"} + } + } } } diff --git a/src/openamp_foundry/benchmark/metrics_snapshot.py b/src/openamp_foundry/benchmark/metrics_snapshot.py index 0b1418f6..2c5652f9 100644 --- a/src/openamp_foundry/benchmark/metrics_snapshot.py +++ b/src/openamp_foundry/benchmark/metrics_snapshot.py @@ -4,16 +4,17 @@ from pathlib import Path +from openamp_foundry.benchmark.feature_decomp import ( + run_feature_decomposition_benchmark, +) from openamp_foundry.benchmark.retrospective import ( run_cluster_split_benchmark, run_expert_ablation_benchmark, run_retrospective_benchmark, run_selectivity_benchmark, ) -from openamp_foundry.benchmark.feature_decomp import ( - run_feature_decomposition_benchmark, -) from openamp_foundry.benchmark.triage import run_strict_triage_benchmark, run_triage_benchmark +from openamp_foundry.selection.ranking_policy import ranking_policy_payload def build_metrics_snapshot( @@ -197,4 +198,9 @@ def build_metrics_snapshot( "per_feature": feature_decomp["per_feature"], "verdict": feature_decomp["verdict"], }, + "ranking_policy": { + "default": ranking_policy_payload("ensemble"), + "alternative": ranking_policy_payload("expert"), + }, + }, } diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index acd7f523..689336d6 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -14,6 +14,7 @@ from openamp_foundry.evidence.schemas import validate_json_schema from openamp_foundry.pipeline import run_ranking_pipeline +from openamp_foundry.selection.ranking_policy import ranking_policy_payload from openamp_foundry.utils.io import read_json @@ -809,7 +810,20 @@ def main(argv: list[str] | None = None) -> int: manifest_path=args.manifest, ranking_mode=getattr(args, "ranking_mode", "ensemble"), ) - print(json.dumps({"status": "ok", "out": args.out, "report": args.report}, indent=2)) + print( + json.dumps( + { + "status": "ok", + "out": args.out, + "report": args.report, + "ranking_mode": getattr(args, "ranking_mode", "ensemble"), + "ranking_policy": ranking_policy_payload( + getattr(args, "ranking_mode", "ensemble") + ), + }, + indent=2, + ) + ) return 0 if args.command == "validate": diff --git a/src/openamp_foundry/pipeline.py b/src/openamp_foundry/pipeline.py index e084afa6..e957aeb0 100644 --- a/src/openamp_foundry/pipeline.py +++ b/src/openamp_foundry/pipeline.py @@ -21,6 +21,7 @@ from openamp_foundry.scoring.selectivity_rich import rich_selectivity_score from openamp_foundry.selection.diversity import greedy_diverse_select from openamp_foundry.selection.pareto import rank_candidates +from openamp_foundry.selection.ranking_policy import ranking_policy_payload from openamp_foundry.types import ScoredCandidate from openamp_foundry.utils.hashing import file_sha256, stable_json_hash from openamp_foundry.utils.io import write_json, write_jsonl @@ -251,6 +252,7 @@ def build_batch_report( }, "selected_ids": sorted(selected_ids), "ranking_mode": ranking_mode, + "ranking_policy": ranking_policy_payload(ranking_mode), } diff --git a/src/openamp_foundry/selection/ranking_policy.py b/src/openamp_foundry/selection/ranking_policy.py new file mode 100644 index 00000000..bb2f0599 --- /dev/null +++ b/src/openamp_foundry/selection/ranking_policy.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass + + +@dataclass(frozen=True) +class RankingPolicy: + mode: str + recommended_for: str + evidence_basis: list[str] + caution: str + not_a_claim: str + + +def get_ranking_policy(mode: str) -> RankingPolicy: + if mode == "expert": + return RankingPolicy( + mode="expert", + recommended_for=( + "Use only as an explicit safety-aware alternative when triaging among " + "AMP-like candidates where hemolysis/selectivity trade-offs matter." + ), + evidence_basis=[ + "Within-AMP selectivity benchmark: expert composite trends better than ensemble on hemolysis-aware ranking, but the signal is not yet statistically secure.", + "Multi-class triage benchmark: ensemble shows anti-selective bias; selectivity_proxy is the only scorer that clearly triages selective > hemolytic > decoy.", + ], + caution=( + "Do not treat expert ranking as the default synthesis gate. On the broader " + "AMP vs decoy benchmark, expert composite underperforms the simple ensemble." + ), + not_a_claim=( + "Expert mode is not a validated safety predictor and does not establish low " + "hemolysis risk. Experimental hemolysis assay is still required." + ), + ) + + return RankingPolicy( + mode="ensemble", + recommended_for=( + "Default synthesis gate for broad AMP-vs-decoy ranking until a stronger " + "selectivity-aware replacement beats it across honest benchmarks." + ), + evidence_basis=[ + "Retrospective AMP vs decoy benchmark: ensemble AUROC remains higher than expert composite.", + "Cluster-split benchmark: ensemble still carries the repo's strongest leakage-aware broad ranking signal.", + ], + caution=( + "Do not mistake the default for safety-awareness. The ensemble shows anti-selective " + "bias on the multi-class triage benchmark and should be checked against safety-specific views." + ), + not_a_claim=( + "Ensemble mode does not prove antimicrobial activity, safety, or wet-lab readiness." + ), + ) + + +def ranking_policy_payload(mode: str) -> dict[str, object]: + return asdict(get_ranking_policy(mode)) diff --git a/tests/test_docs_consistent.py b/tests/test_docs_consistent.py index 3dd4f96b..4fd1b5e4 100644 --- a/tests/test_docs_consistent.py +++ b/tests/test_docs_consistent.py @@ -108,6 +108,14 @@ def test_doc_metrics_snapshot_matches_authoritative_values(self): assert payload["standard"]["n_negatives"] == CURRENT_N_NEGATIVES assert payload["standard"]["n_total"] == CURRENT_N_TOTAL + def test_doc_metrics_snapshot_includes_ranking_policy(self): + """Snapshot must encode the current ranking recommendation contract.""" + payload = json.loads(SNAPSHOT_PATH.read_text(encoding="utf-8")) + assert payload["ranking_policy"]["default"]["mode"] == "ensemble" + assert payload["ranking_policy"]["alternative"]["mode"] == "expert" + assert payload["ranking_policy"]["default"]["evidence_basis"] + assert payload["ranking_policy"]["alternative"]["evidence_basis"] + def test_doc_no_breaking_news_terminology(self): """'breaking news' must be replaced with 'high-impact scenario' in all docs.""" breaking = re.compile(r"breaking[-\s]news", re.IGNORECASE) diff --git a/tests/test_expert_ranking_integration.py b/tests/test_expert_ranking_integration.py index 7374b398..f20be4e8 100644 --- a/tests/test_expert_ranking_integration.py +++ b/tests/test_expert_ranking_integration.py @@ -16,6 +16,8 @@ from __future__ import annotations import json +from io import StringIO +import sys from openamp_foundry.pipeline import score_candidates @@ -96,22 +98,35 @@ def test_cli_expert_ranking_produces_valid_output(self, tmp_path): out = tmp_path / "ranked.jsonl" report = tmp_path / "report.md" - rc = main([ - "rank", - "--candidates", "examples/sequences/demo_candidates.csv", - "--references", "examples/known_reference/demo_known_amps.csv", - "--out", str(out), - "--report", str(report), - "--ranking-mode", "expert", - ]) + old_stdout = sys.stdout + sys.stdout = StringIO() + try: + rc = main([ + "rank", + "--candidates", "examples/sequences/demo_candidates.csv", + "--references", "examples/known_reference/demo_known_amps.csv", + "--out", str(out), + "--report", str(report), + "--ranking-mode", "expert", + ]) + stdout = sys.stdout.getvalue() + finally: + sys.stdout = old_stdout assert rc == 0 assert out.exists() + cli_payload = json.loads(stdout) + assert cli_payload["ranking_mode"] == "expert" + assert cli_payload["ranking_policy"]["mode"] == "expert" + assert "not a validated safety predictor" in cli_payload["ranking_policy"]["not_a_claim"] + # Batch report should record the ranking mode report_json = tmp_path / "report.json" assert report_json.exists() data = json.loads(report_json.read_text()) assert data["ranking_mode"] == "expert" + assert data["ranking_policy"]["mode"] == "expert" + assert len(data["ranking_policy"]["evidence_basis"]) >= 2 def test_cli_ensemble_ranking_default(self, tmp_path): """Default ranking (no --ranking-mode) should be ensemble.""" @@ -130,6 +145,7 @@ def test_cli_ensemble_ranking_default(self, tmp_path): report_json = tmp_path / "report.json" data = json.loads(report_json.read_text()) assert data["ranking_mode"] == "ensemble" + assert data["ranking_policy"]["mode"] == "ensemble" class TestExpertRankingSelectivity: