From 3e0d14e91b5648bb383bcf55fe024ef401abe846 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Mon, 6 Jul 2026 14:59:47 +0700 Subject: [PATCH] =?UTF-8?q?v0.5.57=20=E2=80=94=20Simulation=20Mode=20CLI?= =?UTF-8?q?=20Flag=20(Loop=2037)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 simulation modules are now user-accessible via the rank CLI. Jumped ahead of Loop 36 (cheap-baseline benchmark) — the CLI flag is the higher-leverage bottleneck because it makes modules tangible. Changes: - src/openamp_foundry/cli/main.py -- Added --simulation-mode arg to rank parser with off/info choices. Weighted excluded (gate blocked). - src/openamp_foundry/pipeline.py -- run_ranking_pipeline accepts simulation_mode param. In info mode: runs MembraneProxy + StructureProxy on every candidate, adds sim_* keys to scores dict. batch_report and write_report include simulation data. - tests/test_simulation_mode_cli.py -- 6 tests. - docs/50_LOOP_PLAN.md -- Loop 37 complete, next Loop 36. - docs/ROADMAP.md -- v0.5.57 entry. - docs/METRICS_CURRENT.md -- Pipeline v0.5.57, 1940 tests. 1940 tests passing. --- docs/50_LOOP_PLAN.md | 4 +- docs/METRICS_CURRENT.md | 7 +- docs/ROADMAP.md | 21 +++++ src/openamp_foundry/cli/main.py | 14 ++++ src/openamp_foundry/pipeline.py | 71 ++++++++++++++-- tests/test_simulation_mode_cli.py | 132 ++++++++++++++++++++++++++++++ 6 files changed, 236 insertions(+), 13 deletions(-) create mode 100644 tests/test_simulation_mode_cli.py diff --git a/docs/50_LOOP_PLAN.md b/docs/50_LOOP_PLAN.md index c8172cba..db4be0be 100644 --- a/docs/50_LOOP_PLAN.md +++ b/docs/50_LOOP_PLAN.md @@ -105,7 +105,7 @@ Build the multi-resolution virtual assay layer: structure proxies, membrane inte | 34 ✅ | No within-AMP simulation ablation: do simulation modules improve hemolysis detection? (Original 'membrane reference CSV' deferred — higher leverage to test modules on their actual task first) | Extended `scripts/benchmark_simulation_ablation.py` with `--mode within-amp`. Tests MembraneProxy + StructureProxy on 45 hemolytic vs 125 selective AMPs. Reports per-score detection AUROC. v0.5.55 | 23 tests, 1928 total. Honest finding: best simulation helix_weight AUROC 0.6458; rich_selectivity still best at 0.7453. Simulation modules do NOT improve over existing scorers for hemolysis detection | | 35 ✅ | No executable brake prevented future agents from enabling weighted simulation despite negative ablation results | `simulation/gate.py` + `openamp-foundry bench simulation-gate`: fail-closed permission gate consuming both current ablation artifacts. `make simulation-gate` writes a machine-readable verdict. | 6 tests added; current suite 1927 passing, 7 skipped. Current verdict blocks weighted mode and downgrades to info-only | | 36 | No per-module cheap-baseline benchmark for within-domain simulation value | `benchmark/simulation_baselines.py`: compare each simulation signal against its cheap heuristic baseline on the same task. Report deltas + CI. | Any module with delta ≤ 0 remains permanently experimental | -| 37 | No simulation module that meets the bar to affect selection. All may be experimental | Add `--simulation-mode` flag to the `rank` command: `off` (default, current pipeline), `info` (simulation scores in report only, no ranking impact), `weighted` (simulation affects ranking if benchmarked AUROC > 0.6) | Each mode produces distinct, documented output | +| 37 ✅ | No `--simulation-mode` CLI flag. Simulation modules exist but users cannot access them | Added `--simulation-mode` to `rank` CLI: `off` (default) and `info` (runs MembraneProxy + StructureProxy, adds `sim_*` keys to scores, includes in JSONL/report). Weighted blocked by gate (Loop 35). v0.5.57 | 6 tests, 1940 total. Demo pipeline with `--simulation-mode info` produces sim scores for all 10 candidates. Report includes sim columns | | 38 | No API contract for third-party simulation modules (e.g. Martini MD, AlphaFold) | `simulation/interfaces.py` extended with `ExternalSimulationAdapter` protocol. Documented in ARCHITECTURE.md extension points | Third-party can implement the protocol without reading pipeline internals | | 39 | Virtual-assay benchmark report comparing simulation-augmented vs simulation-free ranking on the strict triage set | `docs/SIMULATION_BENCHMARK.md`: head-to-head comparison of sel_vs_dec, hem_vs_dec, sel_vs_hem AUROCs with and without simulation. If simulation doesn't improve strict triage, say so explicitly | Honest: if delta ≤ 0, report "simulation did not improve triage" | @@ -180,7 +180,7 @@ If no data arrives, virtual assay scaffolding continues independently. Phase 0: ✅ Complete (Loops 1–8) Phase 1: ✅ Complete (Loops 9–17) Phase 2: ✅ Complete (Loops 18–29) -Phase 3: In progress — Loop 35 ✅ (Loops 30–39) +Phase 3: In progress — Loop 37 ✅ (Loops 30–39) Phase 4: Not started (Loops 40–49) ``` diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index cec10a35..9a90dead 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-06 (simulation weighted-mode gate — v0.5.56) +> **Last updated:** 2026-07-06 (simulation mode CLI — v0.5.57) +> **New in v0.5.57:** `rank --simulation-mode info` runs MembraneProxy + StructureProxy on every candidate, adds `sim_*` scores to JSONL output and Markdown report. 6 new CLI tests. 1940 total. > **New in v0.5.56:** `openamp-foundry bench simulation-gate` converts both > simulation ablation artifacts into a fail-closed permission decision for > `weighted` mode. Current verdict: blocked. AMP-vs-decoy delta remains @@ -41,7 +42,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.56 +> **Pipeline version:** v0.5.57 > **Branch:** main --- @@ -1111,7 +1112,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 1927 | +| Total tests | 1940 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e542644f..4974543e 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,5 +1,26 @@ # Roadmap +## v0.5.57 — Simulation Mode CLI Flag (Loop 37) ✓ (2026-07-06) + +Phase 3 simulation modules are now user-accessible via the `rank` CLI. +Jumped ahead of Loop 36 (cheap-baseline benchmark) — the CLI flag is the +higher-leverage bottleneck because it makes the modules tangible. + +Changes: +- ``src/openamp_foundry/cli/main.py`` — Added ``--simulation-mode`` argument + to ``rank`` parser with ``off`` (default) and ``info`` choices. + ``weighted`` excluded — blocked by gate (Loop 35). +- ``src/openamp_foundry/pipeline.py`` — ``run_ranking_pipeline`` accepts + ``simulation_mode`` param. When ``info``: runs MembraneProxy and + StructureProxy on every candidate, adds ``sim_*`` keys to each + ScoredCandidate.scores dict. ``build_batch_report`` includes + ``simulation_mode`` and ``simulation_averages``. ``write_report`` + dynamically adds simulation columns to the Markdown table. +- ``tests/test_simulation_mode_cli.py`` — 6 tests covering: default off + (no sim keys), info adds keys, scores in range, report contains sim + columns, output JSON has simulation_mode field, all candidates have scores. +- 1940 tests passing (1934 existing + 6 new). + ## v0.5.56 — Simulation Weighted-Mode Gate (Loop 35) ✓ (2026-07-06) Phase 3 had a real honesty gap: the scope doc described `weighted` simulation diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index f16138f4..fb9b0f37 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -41,6 +41,17 @@ def build_parser() -> argparse.ArgumentParser: "partially corrects the ensemble's anti-selective bias." ), ) + rank.add_argument( + "--simulation-mode", + choices=["off", "info"], + default="off", + help=( + "Simulation mode: 'off' (default, no simulation scores), " + "'info' (run MembraneProxy and StructureProxy, include " + "scores in report — no ranking impact). 'weighted' mode " + "is not yet available (see 'make simulation-gate')." + ), + ) validate = sub.add_parser("validate", help="Validate a candidate certificate against JSON schema.") validate.add_argument("--certificate", required=True) @@ -1029,6 +1040,7 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.command == "rank": + sim_mode = getattr(args, "simulation_mode", "off") run_ranking_pipeline( candidate_path=args.candidates, reference_path=args.references, @@ -1038,6 +1050,7 @@ def main(argv: list[str] | None = None) -> int: config_path=args.config, manifest_path=args.manifest, ranking_mode=getattr(args, "ranking_mode", "ensemble"), + simulation_mode=sim_mode, ) print( json.dumps( @@ -1046,6 +1059,7 @@ def main(argv: list[str] | None = None) -> int: "out": args.out, "report": args.report, "ranking_mode": getattr(args, "ranking_mode", "ensemble"), + "simulation_mode": sim_mode, "ranking_policy": ranking_policy_payload( getattr(args, "ranking_mode", "ensemble") ), diff --git a/src/openamp_foundry/pipeline.py b/src/openamp_foundry/pipeline.py index e957aeb0..26350d7a 100644 --- a/src/openamp_foundry/pipeline.py +++ b/src/openamp_foundry/pipeline.py @@ -140,11 +140,39 @@ def run_ranking_pipeline( config_path: str | Path = "configs/pipeline.yaml", manifest_path: str | Path | None = None, ranking_mode: str = "ensemble", + simulation_mode: str = "off", ) -> list[ScoredCandidate]: run_id = str(uuid.uuid4()) generated_at = datetime.now(timezone.utc).isoformat() scored, config = score_candidates(candidate_path, reference_path, config_path) + + simulation_keys: list[str] = [] + simulation_averages: dict[str, float] = {} + if simulation_mode in ("info", "weighted"): + from openamp_foundry.simulation.membrane import MembraneProxy + from openamp_foundry.simulation.structure import StructureProxy + membrane = MembraneProxy() + structure = StructureProxy() + sim_keys = [ + "bacterial_binding", "mammalian_binding", "selectivity_ratio", + "helix_weight", "sheet_weight", "coil_weight", "non_helical", + ] + sim_totals: dict[str, float] = {k: 0.0 for k in sim_keys} + for sc in scored: + mem_result = membrane.simulate(sc.candidate.sequence) + struct_result = structure.simulate(sc.candidate.sequence) + for key in ["bacterial_binding", "mammalian_binding", "selectivity_ratio"]: + val = mem_result.scores.get(key, 0.0) + sc.scores[f"sim_{key}"] = val + sim_totals[key] += val + for key in ["helix_weight", "sheet_weight", "coil_weight", "non_helical"]: + val = struct_result.scores.get(key, 0.0) + sc.scores[f"sim_{key}"] = val + sim_totals[key] += val + n = len(scored) if scored else 1 + simulation_keys = [f"sim_{k}" for k in sim_keys] + simulation_averages = {k: round(v / n, 4) for k, v in sim_totals.items()} selection_cfg = config.get("selection", {}) min_novelty = float(selection_cfg.get("min_novelty", 0.0)) max_safety_risk = float(selection_cfg.get("max_safety_risk", 1.0)) @@ -188,8 +216,12 @@ def run_ranking_pipeline( write_json(cert_root / f"{item.candidate.candidate_id}.json", cert) if report_path: - write_report(report_path, ranked, selected) - batch_report = build_batch_report(ranked, selected, generated_at, ranking_mode=ranking_mode) + write_report(report_path, ranked, selected, simulation_keys=simulation_keys) + batch_report = build_batch_report( + ranked, selected, generated_at, ranking_mode=ranking_mode, + simulation_mode=simulation_mode, + simulation_averages=simulation_averages, + ) report_json = Path(report_path).with_suffix(".json") write_json(report_json, batch_report) @@ -223,10 +255,12 @@ def build_batch_report( selected: list[ScoredCandidate], generated_at: str, ranking_mode: str = "ensemble", + simulation_mode: str = "off", + simulation_averages: dict[str, float] | None = None, ) -> dict[str, Any]: """Build a machine-readable batch report validatable against batch_report.schema.json.""" selected_ids = {item.candidate.candidate_id for item in selected} - return { + report: dict[str, Any] = { "pipeline_version": __version__, "candidate_count": len(ranked), "selected_count": len(selected), @@ -254,14 +288,31 @@ def build_batch_report( "ranking_mode": ranking_mode, "ranking_policy": ranking_policy_payload(ranking_mode), } + if simulation_mode != "off" and simulation_averages: + report["simulation_mode"] = simulation_mode + report["simulation_averages"] = simulation_averages + return report def write_report( - path: str | Path, ranked: list[ScoredCandidate], selected: list[ScoredCandidate] + path: str | Path, + ranked: list[ScoredCandidate], + selected: list[ScoredCandidate], + simulation_keys: list[str] | None = None, ) -> None: p = Path(path) p.parent.mkdir(parents=True, exist_ok=True) selected_ids = {item.candidate.candidate_id for item in selected} + sim_keys = simulation_keys or [] + + header = "| Rank | ID | Seq | Ensemble | Activity | Safety | Synthesis | Novelty | Selected |" + sep = "|---:|---|---|---:|---:|---:|---:|---:|:---:|" + if sim_keys: + for k in sim_keys: + display = k.replace("sim_", "Sim ").replace("_", " ") + header = header.replace("| Selected |", f"| {display} | Selected |") + sep = sep.replace("|:---:|", "|---:|:---:|") + lines = [ "# OpenAMP Foundry Candidate Report", "", @@ -273,15 +324,19 @@ def write_report( f"Total candidates scored: {len(ranked)}", f"Candidates selected (passed all filters, diverse): {len(selected)}", "", - "| Rank | ID | Seq | Ensemble | Activity | Safety | Synthesis | Novelty | Selected |", - "|---:|---|---|---:|---:|---:|---:|---:|:---:|", + header, + sep, ] for idx, item in enumerate(ranked, start=1): s = item.scores sel_mark = "Y" if item.candidate.candidate_id in selected_ids else "" - lines.append( + cols = ( f"| {idx} | {item.candidate.candidate_id} | `{item.candidate.sequence}` | " f"{s['ensemble']:.4f} | {s['activity']:.4f} | {s['safety']:.4f} | " - f"{s['synthesis']:.4f} | {s['novelty']:.4f} | {sel_mark} |" + f"{s['synthesis']:.4f} | {s['novelty']:.4f} |" ) + if sim_keys: + cols += " " + " | ".join(f"{s.get(k, 0.0):.4f}" for k in sim_keys) + " |" + cols += f" {sel_mark} |" + lines.append(cols) p.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/tests/test_simulation_mode_cli.py b/tests/test_simulation_mode_cli.py new file mode 100644 index 00000000..0b6cfd93 --- /dev/null +++ b/tests/test_simulation_mode_cli.py @@ -0,0 +1,132 @@ +"""Tests for simulation mode in rank CLI.""" + +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +DEMO_CANDIDATES = "examples/sequences/demo_candidates.csv" +DEMO_REFS = "examples/known_reference/demo_known_amps.csv" + + +def test_simulation_mode_off_default_has_no_sim_scores(): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "ranked.jsonl" + result = subprocess.run( + [sys.executable, "-m", "openamp_foundry.cli", "rank", + "--candidates", DEMO_CANDIDATES, + "--references", DEMO_REFS, + "--out", str(out)], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert result.returncode == 0 + line = out.read_text().splitlines()[0] + data = json.loads(line) + scores = data["scores"] + sim_keys = [k for k in scores if k.startswith("sim_")] + assert len(sim_keys) == 0, f"Expected no sim_ keys, got: {sim_keys}" + + +def test_simulation_mode_info_adds_sim_scores(): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "ranked.jsonl" + result = subprocess.run( + [sys.executable, "-m", "openamp_foundry.cli", "rank", + "--candidates", DEMO_CANDIDATES, + "--references", DEMO_REFS, + "--out", str(out), + "--simulation-mode", "info"], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert result.returncode == 0 + line = out.read_text().splitlines()[0] + data = json.loads(line) + scores = data["scores"] + for key in ["sim_bacterial_binding", "sim_selectivity_ratio", + "sim_helix_weight", "sim_non_helical"]: + assert key in scores, f"Missing simulation key: {key}" + assert isinstance(scores[key], (int, float)) + + +def test_simulation_mode_info_scores_in_range(): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "ranked.jsonl" + result = subprocess.run( + [sys.executable, "-m", "openamp_foundry.cli", "rank", + "--candidates", DEMO_CANDIDATES, + "--references", DEMO_REFS, + "--out", str(out), + "--simulation-mode", "info"], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert result.returncode == 0 + lines = out.read_text().splitlines() + for line in lines: + data = json.loads(line) + scores = data["scores"] + for key in ["sim_bacterial_binding", "sim_selectivity_ratio", + "sim_helix_weight", "sim_coil_weight", "sim_sheet_weight"]: + assert 0.0 <= scores[key] <= 1.5 or key == "sim_selectivity_ratio" and 0.0 <= scores[key] <= 2.0 + + +def test_simulation_mode_info_report_contains_sim(): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "ranked.jsonl" + report = Path(tmp) / "report.md" + result = subprocess.run( + [sys.executable, "-m", "openamp_foundry.cli", "rank", + "--candidates", DEMO_CANDIDATES, + "--references", DEMO_REFS, + "--out", str(out), + "--report", str(report), + "--simulation-mode", "info"], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert result.returncode == 0 + report_text = report.read_text() + assert "Sim bacterial binding" in report_text + assert "Sim helix weight" in report_text + assert "Sim non helical" in report_text + + +def test_simulation_mode_info_output_json_has_simulation_mode(): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "ranked.jsonl" + report = Path(tmp) / "report.md" + result = subprocess.run( + [sys.executable, "-m", "openamp_foundry.cli", "rank", + "--candidates", DEMO_CANDIDATES, + "--references", DEMO_REFS, + "--out", str(out), + "--report", str(report), + "--simulation-mode", "info"], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert result.returncode == 0 + output = json.loads(result.stdout) + assert output["simulation_mode"] == "info" + + +def test_all_candidates_have_sim_scores_in_info_mode(): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "ranked.jsonl" + result = subprocess.run( + [sys.executable, "-m", "openamp_foundry.cli", "rank", + "--candidates", DEMO_CANDIDATES, + "--references", DEMO_REFS, + "--out", str(out), + "--simulation-mode", "info"], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert result.returncode == 0 + lines = out.read_text().splitlines() + for i, line in enumerate(lines): + data = json.loads(line) + assert "sim_bacterial_binding" in data["scores"], f"Candidate {i} missing sim_bacterial_binding"