From 93f9967495f4e590bc131b1f210ec91fecb4e33d Mon Sep 17 00:00:00 2001 From: j Date: Thu, 2 Jul 2026 02:13:00 +0700 Subject: [PATCH] feat: encode ranking policy contract --- docs/ARCHITECTURE.md | 3 + docs/METRICS_CURRENT.md | 1 + docs/ROADMAP.md | 13 +++++ schemas/batch_report.schema.json | 16 ++++- .../benchmark/metrics_snapshot.py | 5 ++ src/openamp_foundry/cli/main.py | 17 +++++- src/openamp_foundry/pipeline.py | 2 + .../selection/ranking_policy.py | 58 +++++++++++++++++++ tests/test_docs_consistent.py | 8 +++ tests/test_expert_ranking_integration.py | 32 +++++++--- 10 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 src/openamp_foundry/selection/ranking_policy.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5f1b9fa1..adaca3b2 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -77,6 +77,9 @@ The system is designed to reduce these failures: 8. Write JSONL results. 9. Generate one certificate per selected candidate. 10. Generate a human-readable report. +11. Record the ranking-policy rationale used for the run so reviewers can see + whether the batch used the broad default gate or the narrower safety-aware + alternative. ## Target future data flow diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index 4f2a8b23..569632c5 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -478,4 +478,5 @@ The MODERATE class (HC50 25-100, n=68) is excluded from the binary task. | 2026-07-01 | Dedicated hemolysis risk scorer: 4-component score (synth+aromatic+face+cys) achieves detection AUROC=0.9218 (CI: 0.82-0.99); integrated into expert composite (detection 0.5119→0.6429); safety scorer unchanged; 1471 tests | OpenAMP loop | | 2026-07-01 | Within-AMP selectivity benchmark added: safety scorer FAILS hemolysis detection (AUROC=0.3844); synthesis is only significant risk detector (AUROC=0.8027); expert composite better than ensemble but not significant (0.5119 vs 0.3486) | OpenAMP loop | | 2026-07-01 | Expert composite ranking integration: `score_candidates()` now computes `expert_composite` and `hemolysis_risk`; `--ranking-mode expert` CLI flag; expert-ranked top-5 have lower mean hemolysis_risk than ensemble | OpenAMP loop | +| 2026-07-02 | Ranking policy contract added: machine-readable recommendation now states `ensemble` remains default broad synthesis gate, `expert` is narrower safety-aware alternative only | OpenAMP loop | | 2026-06-29 | Initial — expanded benchmark (PR #110) | OpenAMP CI | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 34407c09..5085785f 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -233,6 +233,19 @@ ensemble-ranked top-5 on the mixed benchmark set (verified by test). Honest limitation: The expert composite's hemolysis components are not statistically significant detectors on the expanded n=179 benchmark (AUROC=0.565, CI 0.47-0.66). Expert ranking is a safety-aware alternative, not a validated safety guarantee. + +## v0.5.14 — Ranking policy contract ✓ (2026-07-02) + +- Added `selection/ranking_policy.py` — explicit machine-readable policy for when + `ensemble` vs `expert` ranking is justified by current benchmark evidence. +- `rank` CLI now prints the selected ranking policy alongside `ranking_mode`, so + automation can see caution text instead of assuming all ranking modes are equally supported. +- Batch report schema expanded: `report.json` now records `ranking_policy` with + evidence basis, caution, and non-claim language. +- `metrics_snapshot.json` now carries the current default-vs-alternative ranking + recommendation, making benchmark truth easier to consume without re-reading narrative docs. +- Purpose: stop quiet score-shopping. Ensemble remains the broad synthesis gate; + expert mode is a narrower safety-aware alternative that still lacks validated hemolysis prediction. Wet-lab hemolysis assay remains mandatory. ## v1.0 — Validated dry-lab-to-wet-lab loop diff --git a/schemas/batch_report.schema.json b/schemas/batch_report.schema.json index 1ca8eb2d..8c547daf 100644 --- a/schemas/batch_report.schema.json +++ b/schemas/batch_report.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "OpenAMP Batch Report", "type": "object", - "required": ["pipeline_version", "candidate_count", "selected_count", "generated_at", "disclaimer"], + "required": ["pipeline_version", "candidate_count", "selected_count", "generated_at", "disclaimer", "ranking_mode", "ranking_policy"], "properties": { "pipeline_version": {"type": "string"}, "candidate_count": {"type": "integer", "minimum": 0}, @@ -18,6 +18,18 @@ "ensemble": {"type": "number", "minimum": 0, "maximum": 1} } }, - "selected_ids": {"type": "array", "items": {"type": "string"}} + "selected_ids": {"type": "array", "items": {"type": "string"}}, + "ranking_mode": {"type": "string", "enum": ["ensemble", "expert"]}, + "ranking_policy": { + "type": "object", + "required": ["mode", "recommended_for", "evidence_basis", "caution", "not_a_claim"], + "properties": { + "mode": {"type": "string", "enum": ["ensemble", "expert"]}, + "recommended_for": {"type": "string"}, + "evidence_basis": {"type": "array", "items": {"type": "string"}}, + "caution": {"type": "string"}, + "not_a_claim": {"type": "string"} + } + } } } diff --git a/src/openamp_foundry/benchmark/metrics_snapshot.py b/src/openamp_foundry/benchmark/metrics_snapshot.py index bbd431dc..b6d5fcd4 100644 --- a/src/openamp_foundry/benchmark/metrics_snapshot.py +++ b/src/openamp_foundry/benchmark/metrics_snapshot.py @@ -11,6 +11,7 @@ run_selectivity_benchmark, ) from openamp_foundry.benchmark.triage import run_triage_benchmark +from openamp_foundry.selection.ranking_policy import ranking_policy_payload def build_metrics_snapshot( @@ -150,4 +151,8 @@ def build_metrics_snapshot( for scorer, info in triage["per_scorer"].items() }, }, + "ranking_policy": { + "default": ranking_policy_payload("ensemble"), + "alternative": ranking_policy_payload("expert"), + }, } diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index c666c7c2..6d5cf4bf 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -14,6 +14,7 @@ from openamp_foundry.evidence.schemas import validate_json_schema from openamp_foundry.pipeline import run_ranking_pipeline +from openamp_foundry.selection.ranking_policy import ranking_policy_payload from openamp_foundry.utils.io import read_json @@ -670,7 +671,20 @@ def main(argv: list[str] | None = None) -> int: manifest_path=args.manifest, ranking_mode=getattr(args, "ranking_mode", "ensemble"), ) - print(json.dumps({"status": "ok", "out": args.out, "report": args.report}, indent=2)) + print( + json.dumps( + { + "status": "ok", + "out": args.out, + "report": args.report, + "ranking_mode": getattr(args, "ranking_mode", "ensemble"), + "ranking_policy": ranking_policy_payload( + getattr(args, "ranking_mode", "ensemble") + ), + }, + indent=2, + ) + ) return 0 if args.command == "validate": @@ -743,4 +757,3 @@ def main(argv: list[str] | None = None) -> int: if __name__ == "__main__": raise SystemExit(main()) - diff --git a/src/openamp_foundry/pipeline.py b/src/openamp_foundry/pipeline.py index 6a8f7920..eae968fb 100644 --- a/src/openamp_foundry/pipeline.py +++ b/src/openamp_foundry/pipeline.py @@ -20,6 +20,7 @@ from openamp_foundry.scoring.hemolysis import hemolysis_risk_score from openamp_foundry.selection.diversity import greedy_diverse_select from openamp_foundry.selection.pareto import rank_candidates +from openamp_foundry.selection.ranking_policy import ranking_policy_payload from openamp_foundry.types import ScoredCandidate from openamp_foundry.utils.hashing import file_sha256, stable_json_hash from openamp_foundry.utils.io import write_json, write_jsonl @@ -244,6 +245,7 @@ def build_batch_report( }, "selected_ids": sorted(selected_ids), "ranking_mode": ranking_mode, + "ranking_policy": ranking_policy_payload(ranking_mode), } diff --git a/src/openamp_foundry/selection/ranking_policy.py b/src/openamp_foundry/selection/ranking_policy.py new file mode 100644 index 00000000..bb2f0599 --- /dev/null +++ b/src/openamp_foundry/selection/ranking_policy.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass + + +@dataclass(frozen=True) +class RankingPolicy: + mode: str + recommended_for: str + evidence_basis: list[str] + caution: str + not_a_claim: str + + +def get_ranking_policy(mode: str) -> RankingPolicy: + if mode == "expert": + return RankingPolicy( + mode="expert", + recommended_for=( + "Use only as an explicit safety-aware alternative when triaging among " + "AMP-like candidates where hemolysis/selectivity trade-offs matter." + ), + evidence_basis=[ + "Within-AMP selectivity benchmark: expert composite trends better than ensemble on hemolysis-aware ranking, but the signal is not yet statistically secure.", + "Multi-class triage benchmark: ensemble shows anti-selective bias; selectivity_proxy is the only scorer that clearly triages selective > hemolytic > decoy.", + ], + caution=( + "Do not treat expert ranking as the default synthesis gate. On the broader " + "AMP vs decoy benchmark, expert composite underperforms the simple ensemble." + ), + not_a_claim=( + "Expert mode is not a validated safety predictor and does not establish low " + "hemolysis risk. Experimental hemolysis assay is still required." + ), + ) + + return RankingPolicy( + mode="ensemble", + recommended_for=( + "Default synthesis gate for broad AMP-vs-decoy ranking until a stronger " + "selectivity-aware replacement beats it across honest benchmarks." + ), + evidence_basis=[ + "Retrospective AMP vs decoy benchmark: ensemble AUROC remains higher than expert composite.", + "Cluster-split benchmark: ensemble still carries the repo's strongest leakage-aware broad ranking signal.", + ], + caution=( + "Do not mistake the default for safety-awareness. The ensemble shows anti-selective " + "bias on the multi-class triage benchmark and should be checked against safety-specific views." + ), + not_a_claim=( + "Ensemble mode does not prove antimicrobial activity, safety, or wet-lab readiness." + ), + ) + + +def ranking_policy_payload(mode: str) -> dict[str, object]: + return asdict(get_ranking_policy(mode)) diff --git a/tests/test_docs_consistent.py b/tests/test_docs_consistent.py index 11057c14..fea19b7b 100644 --- a/tests/test_docs_consistent.py +++ b/tests/test_docs_consistent.py @@ -103,6 +103,14 @@ def test_doc_metrics_snapshot_matches_authoritative_values(self): assert payload["standard"]["n_negatives"] == CURRENT_N_NEGATIVES assert payload["standard"]["n_total"] == CURRENT_N_TOTAL + def test_doc_metrics_snapshot_includes_ranking_policy(self): + """Snapshot must encode the current ranking recommendation contract.""" + payload = json.loads(SNAPSHOT_PATH.read_text(encoding="utf-8")) + assert payload["ranking_policy"]["default"]["mode"] == "ensemble" + assert payload["ranking_policy"]["alternative"]["mode"] == "expert" + assert payload["ranking_policy"]["default"]["evidence_basis"] + assert payload["ranking_policy"]["alternative"]["evidence_basis"] + def test_doc_no_breaking_news_terminology(self): """'breaking news' must be replaced with 'high-impact scenario' in all docs.""" breaking = re.compile(r"breaking[-\s]news", re.IGNORECASE) diff --git a/tests/test_expert_ranking_integration.py b/tests/test_expert_ranking_integration.py index 7374b398..f20be4e8 100644 --- a/tests/test_expert_ranking_integration.py +++ b/tests/test_expert_ranking_integration.py @@ -16,6 +16,8 @@ from __future__ import annotations import json +from io import StringIO +import sys from openamp_foundry.pipeline import score_candidates @@ -96,22 +98,35 @@ def test_cli_expert_ranking_produces_valid_output(self, tmp_path): out = tmp_path / "ranked.jsonl" report = tmp_path / "report.md" - rc = main([ - "rank", - "--candidates", "examples/sequences/demo_candidates.csv", - "--references", "examples/known_reference/demo_known_amps.csv", - "--out", str(out), - "--report", str(report), - "--ranking-mode", "expert", - ]) + old_stdout = sys.stdout + sys.stdout = StringIO() + try: + rc = main([ + "rank", + "--candidates", "examples/sequences/demo_candidates.csv", + "--references", "examples/known_reference/demo_known_amps.csv", + "--out", str(out), + "--report", str(report), + "--ranking-mode", "expert", + ]) + stdout = sys.stdout.getvalue() + finally: + sys.stdout = old_stdout assert rc == 0 assert out.exists() + cli_payload = json.loads(stdout) + assert cli_payload["ranking_mode"] == "expert" + assert cli_payload["ranking_policy"]["mode"] == "expert" + assert "not a validated safety predictor" in cli_payload["ranking_policy"]["not_a_claim"] + # Batch report should record the ranking mode report_json = tmp_path / "report.json" assert report_json.exists() data = json.loads(report_json.read_text()) assert data["ranking_mode"] == "expert" + assert data["ranking_policy"]["mode"] == "expert" + assert len(data["ranking_policy"]["evidence_basis"]) >= 2 def test_cli_ensemble_ranking_default(self, tmp_path): """Default ranking (no --ranking-mode) should be ensemble.""" @@ -130,6 +145,7 @@ def test_cli_ensemble_ranking_default(self, tmp_path): report_json = tmp_path / "report.json" data = json.loads(report_json.read_text()) assert data["ranking_mode"] == "ensemble" + assert data["ranking_policy"]["mode"] == "ensemble" class TestExpertRankingSelectivity: