From e0b9c7710561629696266142512c500b080223b3 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Mon, 6 Jul 2026 15:28:34 +0700 Subject: [PATCH] =?UTF-8?q?v0.5.60=20=E2=80=94=20add=20simulation=20benchm?= =?UTF-8?q?ark=20report?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + docs/50_LOOP_PLAN.md | 4 +- docs/METRICS_CURRENT.md | 5 +- docs/ROADMAP.md | 15 ++++ docs/SIMULATION_BENCHMARK.md | 73 ++++++++++++++++++++ src/openamp_foundry/simulation/__init__.py | 2 + src/openamp_foundry/simulation/interfaces.py | 1 - tests/test_benchmark_simulation_baselines.py | 2 +- tests/test_docs_consistent.py | 6 ++ 9 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 docs/SIMULATION_BENCHMARK.md diff --git a/README.md b/README.md index 58526231..dc5caa0c 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,7 @@ openamp-foundry/ docs/EVIDENCE_CERTIFICATE.md # candidate certificate spec docs/LEGACY_LOOP_PROMPT.md # reusable execution loop prompt docs/METRICS_CURRENT.md # current benchmark summary + docs/SIMULATION_BENCHMARK.md # virtual-assay benchmark verdict docs/NEW_VISION.md # next-horizon wet-lab compression vision docs/PLAN.md # detailed execution plan docs/ROADMAP.md # shipped milestones and next horizons diff --git a/docs/50_LOOP_PLAN.md b/docs/50_LOOP_PLAN.md index eb1313be..176119ac 100644 --- a/docs/50_LOOP_PLAN.md +++ b/docs/50_LOOP_PLAN.md @@ -107,7 +107,7 @@ Build the multi-resolution virtual assay layer: structure proxies, membrane inte | 36 ✅ | No per-module cheap-baseline benchmark for within-domain simulation value | `scripts/benchmark_simulation_baselines.py`: compares each simulation signal (bacterial_binding, selectivity_ratio, helix_weight, non_helical) against its cheapest heuristic baseline (mean_eisenberg, selectivity_proxy, helix_propensity, proline_fraction). v0.5.58 | 13 tests, 1953 total. Honest finding: 0/4 signals beat baselines. All simulation modules remain permanently experimental. Weighted mode remains blocked | | 37 ✅ | No `--simulation-mode` CLI flag. Simulation modules exist but users cannot access them | Added `--simulation-mode` to `rank` CLI: `off` (default) and `info` (runs MembraneProxy + StructureProxy, adds `sim_*` keys to scores, includes in JSONL/report). Weighted blocked by gate (Loop 35). v0.5.57 | 6 tests, 1940 total. Demo pipeline with `--simulation-mode info` produces sim scores for all 10 candidates. Report includes sim columns | | 38 ✅ | No API contract for third-party simulation modules (e.g. Martini MD, AlphaFold) | `simulation/interfaces.py` extended with `ExternalSimulationAdapter` protocol. ARCHITECTURE.md extension points documented. v0.5.59 | 13 tests, 1966 total. Adapter wraps any callable, checks availability, handles errors, propagates metadata | -| 39 | Virtual-assay benchmark report comparing simulation-augmented vs simulation-free ranking on the strict triage set | `docs/SIMULATION_BENCHMARK.md`: head-to-head comparison of sel_vs_dec, hem_vs_dec, sel_vs_hem AUROCs with and without simulation. If simulation doesn't improve strict triage, say so explicitly | Honest: if delta ≤ 0, report "simulation did not improve triage" | +| 39 ✅ | Virtual-assay benchmark report comparing simulation-augmented vs simulation-free ranking on the strict triage set | `docs/SIMULATION_BENCHMARK.md`: head-to-head report for simulation ablation, cheap-baseline comparison, and weighted-mode gate. v0.5.60. | Honest result: simulation did not improve ranking; weighted mode remains blocked. | **Phase 3 exit criteria:** - At least 2 simulation modules exist and are benchmarked @@ -241,7 +241,7 @@ Phase 4: Not started (Loops 40–49) | 28 ✅ | Policy version bump workflow for when real data arrives | `scripts/bump_recalibration_policy.py`: standalone script with `--dry-run`, decision-log guard, auto-increment + write. CI guard in `ci.yml` validates policy version changes against base branch. v0.5.49. 9 tests. | CI rejects policy PRs without valid decision log; `make bump-policy-version` and `make bump-policy-version-dry-run` available | | 29 ✅ | No public negative-result archive format. If Wave 1 yields all negatives, where do they go? | `docs/NEGATIVE_RESULT_ARCHIVE.md`: full template with entry schema, procedures, automation notes, and limitations. Covers pre-selection rejects, selected-untested, lab inactives, lab toxic, control failures. v0.5.50. | Template complete enough for a lab partner to fill; schema defines 18 fields with required/conditional markers | -**Next loop:** Loop 39 — Phase 3 (virtual-assay benchmark report). +**Next loop:** Loop 40 — lab partner onboarding pack. **Phase 2 exit criteria (all 5 met ✅):** - ✅ `make calibration-loop` runs from clean checkout, produces batch-2 manifest diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index fdf40445..a90f25f8 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-06 (external simulation adapter — v0.5.59) +> **Last updated:** 2026-07-06 (simulation benchmark report — v0.5.60) +> **New in v0.5.60:** `docs/SIMULATION_BENCHMARK.md` consolidates simulation ablation, cheap-baseline comparison, and weighted-mode gate results. Current conclusion: simulation does not improve ranking; `weighted` remains blocked. > **New in v0.5.59:** `ExternalSimulationAdapter` protocol — wraps third-party callables into `VirtualAssayProxy`. Availability check, graceful error handling, metadata override. ARCHITECTURE.md docs updated. 12 tests. 1965 total. > **New in v0.5.58:** Per-signal cheap-baseline comparison: 0/4 simulation signals beat their cheapest heuristic. All simulation modules remain permanently experimental. `make bench-simulation-baselines`. 13 tests. 1953 total. > **New in v0.5.57:** `rank --simulation-mode info` runs MembraneProxy + StructureProxy on every candidate, adds `sim_*` scores to JSONL output and Markdown report. 6 new CLI tests. 1940 total. @@ -44,7 +45,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.59 +> **Pipeline version:** v0.5.60 > **Branch:** main --- diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 27e4af11..e070b87b 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,5 +1,20 @@ # Roadmap +## v0.5.60 — Simulation Benchmark Report (Loop 39) ✓ (2026-07-06) + +Closed the benchmark-report gap with `docs/SIMULATION_BENCHMARK.md`. + +The report consolidates: +- AMP-vs-decoy simulation ablation. +- Within-AMP hemolysis detection ablation. +- Per-signal cheap-baseline comparison. +- Weighted-mode gate decision. + +**Honest finding:** +- Simulation does not improve ranking. +- Weighted mode remains blocked. +- Simulation values may remain visible only in `info` mode. + ## v0.5.59 — External Simulation Adapter Protocol (Loop 38) ✓ (2026-07-06) API contract for third-party simulation modules (Martini MD, AlphaFold, diff --git a/docs/SIMULATION_BENCHMARK.md b/docs/SIMULATION_BENCHMARK.md new file mode 100644 index 00000000..26e519b6 --- /dev/null +++ b/docs/SIMULATION_BENCHMARK.md @@ -0,0 +1,73 @@ +# Simulation Benchmark Report + +## Status + +Current verdict: simulation remains informational only. + +The shipped membrane and structure proxies do not improve candidate ranking +against the repo's current benchmark bar. They may still help humans inspect a +candidate, but they must not affect ranking weights. + +Simulation does not improve ranking on the current benchmark set. + +## Reproducible Commands + +```bash +PYTHONPATH=src python3 scripts/benchmark_simulation_ablation.py \ + --mode amp-vs-decoy --out outputs/simulation_ablation.json || test $? -eq 3 + +PYTHONPATH=src python3 scripts/benchmark_simulation_ablation.py \ + --mode within-amp --out outputs/simulation_ablation_within_amp.json || test $? -eq 3 + +PYTHONPATH=src python3 scripts/benchmark_simulation_baselines.py \ + --out outputs/simulation_baselines.json || test $? -eq 3 + +PYTHONPATH=src python3 -m openamp_foundry.cli bench simulation-gate \ + --amp-vs-decoy-json outputs/simulation_ablation.json \ + --within-amp-json outputs/simulation_ablation_within_amp.json \ + --out outputs/simulation_gate_verdict.json || test $? -eq 3 +``` + +Exit code `3` is expected for benchmarks or gates that correctly reject +weighted simulation. + +## Current Results + +| Benchmark | Baseline / incumbent | Simulation result | Delta | Verdict | +|---|---:|---:|---:|---| +| AMP-vs-decoy ablation | Ensemble AUROC `0.7792` | Ensemble + simulation AUROC `0.6640` | `-0.1153` | No improvement | +| Within-AMP hemolysis detection | Rich selectivity AUROC `0.7453` | Best simulation AUROC `0.6458` | `-0.0995` | No improvement | +| Cheap baseline: bacterial binding | Mean Eisenberg AUROC `0.5469` | Bacterial binding AUROC `0.4876` | `-0.0593` | No improvement | +| Cheap baseline: selectivity ratio | Selectivity proxy AUROC `0.3905` | Selectivity ratio AUROC `0.3615` | `-0.0290` | No improvement | +| Cheap baseline: helix weight | Helix propensity AUROC `0.6489` | Helix weight AUROC `0.6458` | `-0.0031` | No improvement | +| Cheap baseline: non-helical flag | Proline fraction AUROC `0.4929` | Non-helical flag AUROC `0.4124` | `-0.0805` | No improvement | + +## Gate Decision + +`openamp-foundry bench simulation-gate` blocks `weighted` mode. + +Reasons: + +- AMP-vs-decoy ablation does not improve over ensemble. +- Within-AMP ablation does not improve over existing scorers. +- Cheap-baseline comparison shows zero of four simulation signals beat their + simplest meaningful heuristic. + +## Interpretation + +Known fact from current benchmark artifacts: the current simulation layer is +not ready to influence ranking. + +Logical inference: future simulation work needs richer inputs than the current +1D propensities. A more complex module still has to beat cheap baselines before +it earns ranking impact. + +## Policy + +Until this report changes with stronger evidence: + +- `rank --simulation-mode info` is allowed for exploratory reports. +- `weighted` simulation remains blocked. +- Evidence certificates may include simulation values only as experimental + context, not as activity, safety, or selectivity proof. +- No public claim may imply simulation-validated antimicrobial activity. diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index 24057620..14592b34 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -17,4 +17,6 @@ "EmulatorBaseline", "MembraneProxy", "StructureProxy", + "SimulationGateVerdict", + "evaluate_simulation_gate", ] diff --git a/src/openamp_foundry/simulation/interfaces.py b/src/openamp_foundry/simulation/interfaces.py index 6d3e3701..86dc751a 100644 --- a/src/openamp_foundry/simulation/interfaces.py +++ b/src/openamp_foundry/simulation/interfaces.py @@ -2,7 +2,6 @@ import abc import importlib -import sys from dataclasses import dataclass, field from typing import Callable, Dict, List, Optional diff --git a/tests/test_benchmark_simulation_baselines.py b/tests/test_benchmark_simulation_baselines.py index 20e083dc..310efc00 100644 --- a/tests/test_benchmark_simulation_baselines.py +++ b/tests/test_benchmark_simulation_baselines.py @@ -92,7 +92,7 @@ def test_cli_output_contains_verdict(): def test_cli_writes_output_json(tmp_path): out = tmp_path / "baselines.json" - result = subprocess.run( + subprocess.run( [sys.executable, "scripts/benchmark_simulation_baselines.py", "--out", str(out)], capture_output=True, text=True, env={"PYTHONPATH": "src"}, diff --git a/tests/test_docs_consistent.py b/tests/test_docs_consistent.py index a9ce1939..f6725296 100644 --- a/tests/test_docs_consistent.py +++ b/tests/test_docs_consistent.py @@ -108,6 +108,12 @@ def test_doc_metrics_snapshot_matches_authoritative_values(self): assert payload["standard"]["n_negatives"] == CURRENT_N_NEGATIVES assert payload["standard"]["n_total"] == CURRENT_N_TOTAL + def test_doc_simulation_benchmark_report_records_blocked_weighted_mode(self): + report = (DOCS_DIR / "SIMULATION_BENCHMARK.md").read_text(encoding="utf-8") + assert "Current verdict: simulation remains informational only." in report + assert "`weighted` simulation remains blocked." in report + assert "Simulation does not improve ranking" in report + def test_doc_metrics_snapshot_includes_ranking_policy(self): """Snapshot must encode the current ranking recommendation contract.""" payload = json.loads(SNAPSHOT_PATH.read_text(encoding="utf-8"))