From 317dfe6a8b9c344d438be7f305b7232fe61835ce Mon Sep 17 00:00:00 2001 From: OpenCode Date: Mon, 6 Jul 2026 16:10:49 +0700 Subject: [PATCH] feat: expose simulation uncertainty in evidence --- docs/50_LOOP_PLAN.md | 9 +++---- docs/METRICS_CURRENT.md | 9 +++---- docs/ROADMAP.md | 25 ++++++++++++++++++++ docs/SIMULATION_BENCHMARK.md | 3 +++ docs/VIRTUAL_ASSAY_SCOPE.md | 11 ++++++++- schemas/candidate.schema.json | 12 ++++++++++ src/openamp_foundry/pipeline.py | 16 ++++++++++++- tests/test_docs_consistent.py | 9 +++++++ tests/test_simulation_mode_cli.py | 39 +++++++++++++++++++++++++++++-- 9 files changed, 121 insertions(+), 12 deletions(-) diff --git a/docs/50_LOOP_PLAN.md b/docs/50_LOOP_PLAN.md index 4bff65df..196e818f 100644 --- a/docs/50_LOOP_PLAN.md +++ b/docs/50_LOOP_PLAN.md @@ -1,8 +1,9 @@ # 50-Loop Execution Plan -> **Status:** Strategic roadmap. Updated v0.5.56. -> **Current state:** 1927 passing tests, pipeline AUROC 0.7792, calibration pipeline complete (intake + gate + engine + dry-run + policy-version + synthetic generator + recalibration report + batch-2 selector + recovery benchmark + end-to-end integration + pytest golden-path test + policy bump workflow + negative result archive), +> **Status:** Strategic roadmap. Updated v0.5.62. +> **Current state:** 1961 passing tests, pipeline AUROC 0.7792, calibration pipeline complete (intake + gate + engine + dry-run + policy-version + synthetic generator + recalibration report + batch-2 selector + recovery benchmark + end-to-end integration + pytest golden-path test + policy bump workflow + negative result archive), > > Phase 0 complete (Loops 1–8), Phase 1 complete (Loops 9–17), Phase 2 complete (Loops 18–29), +> Phase 3 complete (Loops 30–39 + v0.5.62 uncertainty evidence hardening), > cluster-split/selectivity/triage now gated in CI, Wave 0.5 panel ready (24 candidates, 15 families), no wet-lab data yet. > > Each loop = one focused PR: bottleneck identified → implemented → verified → merged. @@ -92,7 +93,7 @@ Build the recalibration engine gated by the policy. Implement active-learning ba --- -## Phase 3 — Virtual Assay Scaffolding (Loops 30–39) +## Phase 3 — Virtual Assay Scaffolding (Loops 30–40) ✅ Build the multi-resolution virtual assay layer: structure proxies, membrane interaction models, uncertainty-aware surrogates. Every module must justify itself against cheap heuristics. @@ -118,7 +119,7 @@ Build the multi-resolution virtual assay layer: structure proxies, membrane inte --- -## Phase 4 — Wet-Lab Readiness & External Validation (Loops 40–49) +## Phase 4 — Wet-Lab Readiness & External Validation (Loops 41–50) Prepare for real wet-lab partners. Make the pack reviewable, the evidence traceable, and the process pre-registered. diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index 664ede7f..ea11178f 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -5,8 +5,9 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-06 (lab partner onboarding — v0.5.61) -> **New in v0.5.61:** `docs/LAB_PARTNER_ONBOARDING.md` — CRO onboarding pack with panel summary, synthesis instructions, assay protocols, data return format, controls, safety, timeline. No code changes. 1966 tests. +> **Last updated:** 2026-07-06 (simulation uncertainty in evidence — v0.5.62) +> **New in v0.5.62:** `rank --simulation-mode info` now propagates `sim_membrane_uncertainty`, `sim_structure_uncertainty`, and `sim_max_uncertainty` into ranked outputs and selected-candidate evidence certificates. This is reviewability only; `weighted` remains blocked. Current suite: 1961 passed, 7 skipped. +> **New in v0.5.61:** `docs/LAB_PARTNER_ONBOARDING.md` — CRO onboarding pack with panel summary, synthesis instructions, assay protocols, data return format, controls, safety, timeline. No code changes. > **New in v0.5.60:** `docs/SIMULATION_BENCHMARK.md` consolidates simulation ablation, cheap-baseline comparison, and weighted-mode gate results. Current conclusion: simulation does not improve ranking; `weighted` remains blocked. > **New in v0.5.59:** `ExternalSimulationAdapter` protocol — wraps third-party callables into `VirtualAssayProxy`. Availability check, graceful error handling, metadata override. ARCHITECTURE.md docs updated. 12 tests. 1965 total. > **New in v0.5.58:** Per-signal cheap-baseline comparison: 0/4 simulation signals beat their cheapest heuristic. All simulation modules remain permanently experimental. `make bench-simulation-baselines`. 13 tests. 1953 total. @@ -46,7 +47,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.60 +> **Pipeline version:** v0.5.62 > **Branch:** main --- @@ -1116,7 +1117,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 1965 | +| Total tests | 1961 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 7900a0d6..0adf41df 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,5 +1,30 @@ # Roadmap +## v0.5.62 — Simulation Uncertainty in Evidence ✓ (2026-07-06) + +Closed the remaining Phase 3 evidence-propagation gap: simulation uncertainty +now travels with selected candidates instead of living only inside proxy return +objects. + +Changes: +- ``src/openamp_foundry/pipeline.py`` — `rank --simulation-mode info` now + writes three informational uncertainty fields into candidate scores: + ``sim_membrane_uncertainty``, ``sim_structure_uncertainty``, and + ``sim_max_uncertainty``. Candidates above the `0.5` uncertainty bar gain an + explicit failure-mode note stating that simulation outputs must not affect + selection. +- ``schemas/candidate.schema.json`` — documents the three uncertainty keys so + evidence certificates can carry them without schema drift. +- ``tests/test_simulation_mode_cli.py`` — covers ranked output, report columns, + evidence-certificate inclusion, and schema validation for the new fields. +- ``docs/VIRTUAL_ASSAY_SCOPE.md`` and ``docs/SIMULATION_BENCHMARK.md`` — synced + to current behavior. + +Honest finding: +- This does not improve ranking. +- It improves reviewability: virtual-assay outputs now carry their own + uncertainty wherever selected-candidate evidence is consumed. + ## v0.5.61 — Lab Partner Onboarding Pack (Loop 40) ✓ (2026-07-06) Phase 4 begins with a comprehensive onboarding document for CRO or academic diff --git a/docs/SIMULATION_BENCHMARK.md b/docs/SIMULATION_BENCHMARK.md index 87b8011e..a6704c7c 100644 --- a/docs/SIMULATION_BENCHMARK.md +++ b/docs/SIMULATION_BENCHMARK.md @@ -125,6 +125,9 @@ Until this report changes with stronger evidence: - `weighted` simulation remains blocked. - Evidence certificates may include simulation values only as experimental context, not as activity, safety, or selectivity proof. +- Evidence certificates now include `sim_membrane_uncertainty`, + `sim_structure_uncertainty`, and `sim_max_uncertainty` when simulation + info mode is used. - No public claim may imply simulation-validated antimicrobial activity. - Any future simulation module must pass the gate (Loop 35) before affecting ranking. The gate will not be bypassed even if the module appears plausible. diff --git a/docs/VIRTUAL_ASSAY_SCOPE.md b/docs/VIRTUAL_ASSAY_SCOPE.md index 68f4fbdd..5a54c50e 100644 --- a/docs/VIRTUAL_ASSAY_SCOPE.md +++ b/docs/VIRTUAL_ASSAY_SCOPE.md @@ -18,6 +18,9 @@ have explicit ablation evidence. Current state: `rich_selectivity` on hemolysis detection. - `bench simulation-gate` therefore blocks `weighted` mode and downgrades to `info` until the evidence changes. +- `rank --simulation-mode info` propagates membrane, structure, and maximum + simulation uncertainty into ranked JSONL rows, Markdown reports, and evidence + certificates. These values are informational only. ## Core Principle @@ -70,6 +73,12 @@ surface that directly. A module with `uncertainty > 0.5` (on a 0–1 scale where 0 = fully constrained, 1 = unconstrained) is experimental and must not affect selection. +When `rank --simulation-mode info` is used, selected-candidate evidence +certificates include `sim_membrane_uncertainty`, +`sim_structure_uncertainty`, and `sim_max_uncertainty`. These fields expose +proxy uncertainty for reviewers; they do not change ranking, filtering, or +selection. + ## Ablation Requirement Each module must pass an ablation benchmark before it can affect @@ -149,6 +158,6 @@ Phase 3 is complete when: `weighted` mode. 3. Modules that fail the ablation are either removed or permanently marked experimental. -4. Uncertainty is propagated through to the evidence certificate. +4. Uncertainty is propagated through to the evidence certificate. ✅ 5. The external adapter protocol (`ExternalSimulationAdapter`) is documented in ARCHITECTURE.md. diff --git a/schemas/candidate.schema.json b/schemas/candidate.schema.json index 16c113bf..28b285c2 100644 --- a/schemas/candidate.schema.json +++ b/schemas/candidate.schema.json @@ -38,6 +38,18 @@ "disagreement": { "type": "number", "minimum": 0, "maximum": 1, "description": "Absolute difference between activity_likeness and boman_activity. High values indicate scoring uncertainty." + }, + "sim_membrane_uncertainty": { + "type": "number", "minimum": 0, "maximum": 1, + "description": "Experimental membrane-proxy uncertainty. Informational only unless weighted simulation is explicitly allowed." + }, + "sim_structure_uncertainty": { + "type": "number", "minimum": 0, "maximum": 1, + "description": "Experimental structure-proxy uncertainty. Informational only unless weighted simulation is explicitly allowed." + }, + "sim_max_uncertainty": { + "type": "number", "minimum": 0, "maximum": 1, + "description": "Maximum virtual-assay proxy uncertainty for this candidate. Values above 0.5 must not affect selection." } } }, diff --git a/src/openamp_foundry/pipeline.py b/src/openamp_foundry/pipeline.py index 26350d7a..79a720b5 100644 --- a/src/openamp_foundry/pipeline.py +++ b/src/openamp_foundry/pipeline.py @@ -156,7 +156,9 @@ def run_ranking_pipeline( structure = StructureProxy() sim_keys = [ "bacterial_binding", "mammalian_binding", "selectivity_ratio", - "helix_weight", "sheet_weight", "coil_weight", "non_helical", + "membrane_uncertainty", "helix_weight", "sheet_weight", + "coil_weight", "non_helical", "structure_uncertainty", + "max_uncertainty", ] sim_totals: dict[str, float] = {k: 0.0 for k in sim_keys} for sc in scored: @@ -166,10 +168,22 @@ def run_ranking_pipeline( val = mem_result.scores.get(key, 0.0) sc.scores[f"sim_{key}"] = val sim_totals[key] += val + sc.scores["sim_membrane_uncertainty"] = mem_result.uncertainty + sim_totals["membrane_uncertainty"] += mem_result.uncertainty for key in ["helix_weight", "sheet_weight", "coil_weight", "non_helical"]: val = struct_result.scores.get(key, 0.0) sc.scores[f"sim_{key}"] = val sim_totals[key] += val + sc.scores["sim_structure_uncertainty"] = struct_result.uncertainty + sim_totals["structure_uncertainty"] += struct_result.uncertainty + max_uncertainty = max(mem_result.uncertainty, struct_result.uncertainty) + sc.scores["sim_max_uncertainty"] = max_uncertainty + sim_totals["max_uncertainty"] += max_uncertainty + if max_uncertainty > 0.5: + sc.known_failure_modes.append( + "Simulation proxy uncertainty exceeds 0.5; simulation outputs " + "must not affect selection." + ) n = len(scored) if scored else 1 simulation_keys = [f"sim_{k}" for k in sim_keys] simulation_averages = {k: round(v / n, 4) for k, v in sim_totals.items()} diff --git a/tests/test_docs_consistent.py b/tests/test_docs_consistent.py index f6725296..cf6967f8 100644 --- a/tests/test_docs_consistent.py +++ b/tests/test_docs_consistent.py @@ -113,6 +113,15 @@ def test_doc_simulation_benchmark_report_records_blocked_weighted_mode(self): assert "Current verdict: simulation remains informational only." in report assert "`weighted` simulation remains blocked." in report assert "Simulation does not improve ranking" in report + assert "sim_max_uncertainty" in report + + def test_virtual_assay_scope_records_uncertainty_certificate_contract(self): + scope = (DOCS_DIR / "VIRTUAL_ASSAY_SCOPE.md").read_text(encoding="utf-8") + assert "evidence" in scope + assert "certificates" in scope + assert "sim_membrane_uncertainty" in scope + assert "sim_structure_uncertainty" in scope + assert "sim_max_uncertainty" in scope def test_doc_metrics_snapshot_includes_ranking_policy(self): """Snapshot must encode the current ranking recommendation contract.""" diff --git a/tests/test_simulation_mode_cli.py b/tests/test_simulation_mode_cli.py index 0b6cfd93..dcb65d8b 100644 --- a/tests/test_simulation_mode_cli.py +++ b/tests/test_simulation_mode_cli.py @@ -6,8 +6,11 @@ import tempfile from pathlib import Path +from openamp_foundry.evidence.schemas import validate_json_schema + DEMO_CANDIDATES = "examples/sequences/demo_candidates.csv" DEMO_REFS = "examples/known_reference/demo_known_amps.csv" +CERT_SCHEMA = Path(__file__).parents[1] / "schemas" / "candidate.schema.json" def test_simulation_mode_off_default_has_no_sim_scores(): @@ -46,7 +49,9 @@ def test_simulation_mode_info_adds_sim_scores(): data = json.loads(line) scores = data["scores"] for key in ["sim_bacterial_binding", "sim_selectivity_ratio", - "sim_helix_weight", "sim_non_helical"]: + "sim_helix_weight", "sim_non_helical", + "sim_membrane_uncertainty", "sim_structure_uncertainty", + "sim_max_uncertainty"]: assert key in scores, f"Missing simulation key: {key}" assert isinstance(scores[key], (int, float)) @@ -69,7 +74,9 @@ def test_simulation_mode_info_scores_in_range(): data = json.loads(line) scores = data["scores"] for key in ["sim_bacterial_binding", "sim_selectivity_ratio", - "sim_helix_weight", "sim_coil_weight", "sim_sheet_weight"]: + "sim_helix_weight", "sim_coil_weight", "sim_sheet_weight", + "sim_membrane_uncertainty", "sim_structure_uncertainty", + "sim_max_uncertainty"]: assert 0.0 <= scores[key] <= 1.5 or key == "sim_selectivity_ratio" and 0.0 <= scores[key] <= 2.0 @@ -92,6 +99,7 @@ def test_simulation_mode_info_report_contains_sim(): assert "Sim bacterial binding" in report_text assert "Sim helix weight" in report_text assert "Sim non helical" in report_text + assert "Sim max uncertainty" in report_text def test_simulation_mode_info_output_json_has_simulation_mode(): @@ -130,3 +138,30 @@ def test_all_candidates_have_sim_scores_in_info_mode(): for i, line in enumerate(lines): data = json.loads(line) assert "sim_bacterial_binding" in data["scores"], f"Candidate {i} missing sim_bacterial_binding" + + +def test_simulation_mode_info_evidence_certificates_include_uncertainty(): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "ranked.jsonl" + certs = Path(tmp) / "certs" + result = subprocess.run( + [sys.executable, "-m", "openamp_foundry.cli", "rank", + "--candidates", DEMO_CANDIDATES, + "--references", DEMO_REFS, + "--out", str(out), + "--cert-dir", str(certs), + "--simulation-mode", "info"], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + + assert result.returncode == 0 + cert_paths = sorted(certs.glob("*.json")) + assert cert_paths, "Expected selected-candidate certificates" + cert = json.loads(cert_paths[0].read_text()) + validate_json_schema(cert, CERT_SCHEMA) + scores = cert["scores"] + assert "sim_membrane_uncertainty" in scores + assert "sim_structure_uncertainty" in scores + assert "sim_max_uncertainty" in scores + assert 0.0 <= scores["sim_max_uncertainty"] <= 1.0