v0.5.56 — gate simulation weighted mode

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version
+.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version
 
 PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
 PYTEST  := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
@@ -41,6 +41,9 @@ help:
 	@echo "  make bench-order-dependent    Analyze which features survive scrambling (order-dependence)"
 	@echo "  make bench-precision-at-k    Precision@k operating characteristic (small-k triage, threshold calibration)"
 	@echo "  make bench-per-family        Per-family benchmark — stratify 500 AMPs by structural class, per-class AUROC"
+	@echo "  make bench-simulation-ablation  Simulation ablation on AMP-vs-decoy benchmark"
+	@echo "  make bench-simulation-ablation-within-amp  Simulation ablation on hemolytic vs selective AMPs"
+	@echo "  make simulation-gate        Fail-closed gate for weighted simulation mode"
 	@echo "  make bench-gate               Benchmark regression gate (AUROC drift check)"
 	@echo "  make regenerate-all           Run all pipeline + benchmarks and verify determinism"
 	@echo "  make bench-hidden-active      Hidden-positive recovery on mixed benchmark set"
@@ -445,6 +448,12 @@ bench-simulation-ablation-within-amp:
 	PYTHONPATH=src $(PYTHON) scripts/benchmark_simulation_ablation.py \
 		--mode within-amp --out outputs/simulation_ablation_within_amp.json
 
+simulation-gate:
+	PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench simulation-gate \
+		--amp-vs-decoy-json outputs/simulation_ablation.json \
+		--within-amp-json outputs/simulation_ablation_within_amp.json \
+		--out outputs/simulation_gate_verdict.json
+
 bench-cross-dataset:
 	PYTHONPATH=src $(PYTHON) scripts/benchmark_cross_dataset.py \
 		--out outputs/cross_dataset_benchmark.json

docs/50_LOOP_PLAN.md

@@ -1,7 +1,7 @@
 # 50-Loop Execution Plan
 
-> **Status:** Strategic roadmap. Updated v0.5.50.
-> **Current state:** 1843 tests, pipeline AUROC 0.7792, calibration pipeline complete (intake + gate + engine + dry-run + policy-version + synthetic generator + recalibration report + batch-2 selector + recovery benchmark + end-to-end integration + pytest golden-path test + policy bump workflow + negative result archive),
+> **Status:** Strategic roadmap. Updated v0.5.56.
+> **Current state:** 1927 passing tests, pipeline AUROC 0.7792, calibration pipeline complete (intake + gate + engine + dry-run + policy-version + synthetic generator + recalibration report + batch-2 selector + recovery benchmark + end-to-end integration + pytest golden-path test + policy bump workflow + negative result archive),
 > > Phase 0 complete (Loops 1–8), Phase 1 complete (Loops 9–17), Phase 2 complete (Loops 18–29),
 > cluster-split/selectivity/triage now gated in CI, Wave 0.5 panel ready (24 candidates, 15 families), no wet-lab data yet.
 >
@@ -103,8 +103,8 @@ Build the multi-resolution virtual assay layer: structure proxies, membrane inte
 | 32 ✅ | No structure ensemble proxy. Current pipeline assumes all AMPs are helical | `simulation/structure.py`: StructureProxy with Chou-Fasman 3-state propensities, normalized ensemble weights, non_helical flag. v0.5.53 | 34 tests, 1905 total. Protegrin/tachyplesin non-helical (✅), magainin not non-helical (✅). Melittin ambiguous — Chou-Fasman limitation documented |
 | 33 ✅ | No per-module ablation: do simulation modules improve discrimination? (Original 'selectivity ratio' superseded by Loop 31 membrane proxy — replaced with more useful ablation benchmark) | `scripts/benchmark_simulation_ablation.py`: tests MembraneProxy + StructureProxy on 500-AMP AMP-vs-decoy benchmark. Reports per-module AUROC, combined composite AUROC, and verdict. v0.5.54 | 17 tests, 1922 total. Honest finding: composite degrades AMP-vs-decoy (delta=-0.1153), but bacterial_binding alone achieves 0.7512 AUROC — genuine non-charge signal from Wimley-White + hydrophobic moment |
 | 34 ✅ | No within-AMP simulation ablation: do simulation modules improve hemolysis detection? (Original 'membrane reference CSV' deferred — higher leverage to test modules on their actual task first) | Extended `scripts/benchmark_simulation_ablation.py` with `--mode within-amp`. Tests MembraneProxy + StructureProxy on 45 hemolytic vs 125 selective AMPs. Reports per-score detection AUROC. v0.5.55 | 23 tests, 1928 total. Honest finding: best simulation helix_weight AUROC 0.6458; rich_selectivity still best at 0.7453. Simulation modules do NOT improve over existing scorers for hemolysis detection |
-| 35 | No per-module ablation: each simulation module must beat a cheap heuristic baseline before affecting selection | `benchmark/simulation_ablation.py`: compares each simulation module against trivial baseline (single-feature proxy). Reports delta AUROC for each | Any module with delta ≤ 0 is flagged as `NO_IMPROVEMENT` and disabled from production pipeline |
-| 36 | No uncertainty-aware composite that folds simulation outputs into the ranking | `scoring/simulation_composite.py`: weighted combination of simulation modules, weighted by their benchmarked AUROC. Uncertainty from `SimulationResult` propagates to final score | Uncertainty reported alongside every candidate score in evidence certificate |
+| 35 ✅ | No executable brake prevented future agents from enabling weighted simulation despite negative ablation results | `simulation/gate.py` + `openamp-foundry bench simulation-gate`: fail-closed permission gate consuming both current ablation artifacts. `make simulation-gate` writes a machine-readable verdict. | 6 tests added; current suite 1927 passing, 7 skipped. Current verdict blocks weighted mode and downgrades to info-only |
+| 36 | No per-module cheap-baseline benchmark for within-domain simulation value | `benchmark/simulation_baselines.py`: compare each simulation signal against its cheap heuristic baseline on the same task. Report deltas + CI. | Any module with delta ≤ 0 remains permanently experimental |
 | 37 | No simulation module that meets the bar to affect selection. All may be experimental | Add `--simulation-mode` flag to the `rank` command: `off` (default, current pipeline), `info` (simulation scores in report only, no ranking impact), `weighted` (simulation affects ranking if benchmarked AUROC > 0.6) | Each mode produces distinct, documented output |
 | 38 | No API contract for third-party simulation modules (e.g. Martini MD, AlphaFold) | `simulation/interfaces.py` extended with `ExternalSimulationAdapter` protocol. Documented in ARCHITECTURE.md extension points | Third-party can implement the protocol without reading pipeline internals |
 | 39 | Virtual-assay benchmark report comparing simulation-augmented vs simulation-free ranking on the strict triage set | `docs/SIMULATION_BENCHMARK.md`: head-to-head comparison of sel_vs_dec, hem_vs_dec, sel_vs_hem AUROCs with and without simulation. If simulation doesn't improve strict triage, say so explicitly | Honest: if delta ≤ 0, report "simulation did not improve triage" |
@@ -180,7 +180,7 @@ If no data arrives, virtual assay scaffolding continues independently.
 Phase 0: ✅ Complete (Loops 1–8)
 Phase 1: ✅ Complete (Loops 9–17)
 Phase 2: ✅ Complete (Loops 18–29)
-Phase 3: In progress — Loop 34 ✅ (Loops 30–39)
+Phase 3: In progress — Loop 35 ✅ (Loops 30–39)
 Phase 4: Not started (Loops 40–49)
 ```
 
@@ -241,7 +241,7 @@ Phase 4: Not started (Loops 40–49)
 | 28 ✅ | Policy version bump workflow for when real data arrives | `scripts/bump_recalibration_policy.py`: standalone script with `--dry-run`, decision-log guard, auto-increment + write. CI guard in `ci.yml` validates policy version changes against base branch. v0.5.49. 9 tests. | CI rejects policy PRs without valid decision log; `make bump-policy-version` and `make bump-policy-version-dry-run` available |
 | 29 ✅ | No public negative-result archive format. If Wave 1 yields all negatives, where do they go? | `docs/NEGATIVE_RESULT_ARCHIVE.md`: full template with entry schema, procedures, automation notes, and limitations. Covers pre-selection rejects, selected-untested, lab inactives, lab toxic, control failures. v0.5.50. | Template complete enough for a lab partner to fill; schema defines 18 fields with required/conditional markers |
 
-**Next loop:** Loop 35 — Phase 3 (membrane reference benchmark set or simulation-mode CLI flag).
+**Next loop:** Loop 36 — Phase 3 cheap-baseline benchmark for each simulation signal.
 
 **Phase 2 exit criteria (all 5 met ✅):**
 - ✅ `make calibration-loop` runs from clean checkout, produces batch-2 manifest

docs/BENCHMARKING.md

@@ -73,7 +73,7 @@ documented result in `docs/METRICS_CURRENT.md` and
 | Benchmark | Purpose | Reason deferred |
 |-----------|---------|----------------|
 | Time split | Test future generalization | Metadata not available for most sequences |
-| Virtual-assay ablation | Test whether simulation modules improve triage | Phase 3 — no modules exist yet |
+| Virtual-assay ablation | Test whether simulation modules improve triage | Shipped: `make bench-simulation-ablation`, `make bench-simulation-ablation-within-amp`, `make simulation-gate` |
 | Biologically plausible charge-balanced negative set | Honest non-charge signal test | Requires curated negative sequences — see v0.5.41 recommendation |
 
 ## Minimum report fields

docs/METRICS_CURRENT.md

@@ -5,7 +5,12 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
 > with this file, this file wins. Updated whenever benchmark/benchmark config changes.
 >
-> **Last updated:** 2026-07-06 (within-AMP simulation ablation — v0.5.55)
+> **Last updated:** 2026-07-06 (simulation weighted-mode gate — v0.5.56)
+> **New in v0.5.56:** `openamp-foundry bench simulation-gate` converts both
+> simulation ablation artifacts into a fail-closed permission decision for
+> `weighted` mode. Current verdict: blocked. AMP-vs-decoy delta remains
+> negative and within-AMP delta remains negative, so simulation stays
+> informational only. 6 tests. Current suite: 1927 passed, 7 skipped.
 > **New in v0.5.55:** Within-AMP simulation ablation — `--mode within-amp` tests MembraneProxy + StructureProxy on hemolysis detection (45 hemolytic vs 125 selective). Honest finding: best simulation `helix_weight` AUROC 0.6458; `rich_selectivity` still best at 0.7453. 23 tests. 1928 total.
 > **New in v0.5.54:** Simulation ablation benchmark — tests MembraneProxy + StructureProxy on 500-AMP AMP-vs-decoy benchmark. Honest finding: composite degrades (delta=−0.1153) but `bacterial_binding` alone achieves AUROC 0.7512 — genuine non-charge signal. 17 new tests. 1922 total.
 > **New in v0.5.53:** Structure ensemble proxy — `StructureProxy` using Chou-Fasman 3-state helix/sheet/coil propensities. `non_helical` flag for helic-biased scorer warnings. `HelicityBaseline`. 34 new tests. 1905 total.
@@ -36,7 +41,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test).
 > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination.
 > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison.
-> **Pipeline version:** v0.5.55
+> **Pipeline version:** v0.5.56
 > **Branch:** main
 
 ---
@@ -1106,7 +1111,7 @@ Decoys score low on activity. Selective AMPs score moderately on both.
 
 | Metric | Value |
 |--------|-------|
-| Total tests | 1928 |
+| Total tests | 1927 |
 | Coverage (branch) | 99% (6 CLI guard lines only) |
 | Source modules at 100% | All pipeline, QC, scoring modules |
 

docs/ROADMAP.md

@@ -1,5 +1,33 @@
 # Roadmap
 
+## v0.5.56 — Simulation Weighted-Mode Gate (Loop 35) ✓ (2026-07-06)
+
+Phase 3 had a real honesty gap: the scope doc described `weighted` simulation
+mode, but no executable guard converted ablation evidence into an allow/block
+decision. That left future agents one edit away from integration theater.
+
+Changes:
+- ``src/openamp_foundry/simulation/gate.py`` — fail-closed gate that consumes
+  current AMP-vs-decoy and within-AMP ablation artifacts and returns a
+  machine-readable permission verdict for simulation integration mode.
+- ``openamp-foundry bench simulation-gate`` — CLI wrapper with exit code `0`
+  only when weighted mode is honestly allowed, `3` otherwise.
+- ``Makefile`` — ``make simulation-gate`` target writing
+  ``outputs/simulation_gate_verdict.json`` from both ablation artifacts.
+- ``docs/VIRTUAL_ASSAY_SCOPE.md`` — corrected stale claim that `rank` already
+  accepts `--simulation-mode`; current state now says weighted mode is blocked
+  until the gate passes.
+- ``tests/test_simulation_gate.py`` — 6 tests covering blocked current-state
+  verdicts, positive-path permission, fail-closed missing artifacts, and CLI
+  exit behavior.
+- Full verification after this loop: ``1927 passed, 7 skipped``.
+
+**Honest finding:**
+- Weighted simulation remains blocked. Current ablations do not justify ranking
+  impact.
+- This is not a modeling advance. It is a truthfulness and architecture
+  advance: the repo now has an executable brake against simulation theater.
+
 ## v0.5.55 — Within-AMP Simulation Ablation (Loop 34) ✓ (2026-07-06)
 
 Extended the simulation ablation benchmark to test modules on their

docs/VIRTUAL_ASSAY_SCOPE.md

@@ -9,9 +9,15 @@ an impressive-looking model that does not improve decision quality is noise.
 
 ## Status
 
-**Phase 3 — Planning.** No virtual-assay modules are implemented in the
-production pipeline. This document describes the intended scope and
-constraints. Implementation starts with Loop 31.
+**Phase 3 — experimental modules implemented, production integration still gated.**
+`MembraneProxy` (Loop 31) and `StructureProxy` (Loop 32) exist, and both now
+have explicit ablation evidence. Current state:
+
+- `bench-simulation-ablation` shows simulation composite degrades AMP-vs-decoy.
+- `bench-simulation-ablation-within-amp` shows simulation scores do not beat
+  `rich_selectivity` on hemolysis detection.
+- `bench simulation-gate` therefore blocks `weighted` mode and downgrades to
+  `info` until the evidence changes.
 
 ## Core Principle
 
@@ -88,9 +94,12 @@ The virtual assay layer supports three modes:
 |------|--------|-------------|
 | `off` (default) | No simulation scores computed. Current pipeline. | Production candidate ranking |
 | `info` | Simulation scores computed and included in reports only. No ranking impact. | Benchmarking, exploration |
-| `weighted` | Simulation scores affect ranking if each module clears its ablation bar. | Only after ablation validation |
+| `weighted` | Simulation scores affect ranking only if the simulation gate passes on current ablation artifacts. | Only after ablation validation |
 
-The `rank` CLI accepts `--simulation-mode` to select the mode.
+The `rank` CLI does **not** yet expose `--simulation-mode`. That omission is
+intentional until a module clears the gate honestly. Current executable policy:
+`openamp-foundry bench simulation-gate` may return `weighted` permission in the
+future, but production ranking remains simulation-free today.
 
 ## Calibration Requirement
 

src/openamp_foundry/active_learning/benchmark.py

@@ -22,7 +22,7 @@
 import csv
 import logging
 import random
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 

src/openamp_foundry/active_learning/selector.py

@@ -9,7 +9,7 @@
 
 import csv
 import logging
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from pathlib import Path
 
 _log = logging.getLogger(__name__)

src/openamp_foundry/cli/commands/benchmark.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 import argparse
 import json
+from pathlib import Path
 
 def _run_bench(args: argparse.Namespace) -> int:
     from openamp_foundry.data.loaders import load_candidates_csv
@@ -320,3 +321,19 @@ def _run_active_learning_bench(args: argparse.Namespace) -> int:
     out_dict = result.to_dict()
     print(json.dumps(out_dict, indent=2))
     return 0
+
+
+def _run_simulation_gate(args: argparse.Namespace) -> int:
+    """Validate whether virtual-assay scores may affect ranking."""
+    from openamp_foundry.simulation.gate import evaluate_simulation_gate
+
+    verdict = evaluate_simulation_gate(
+        amp_vs_decoy_path=args.amp_vs_decoy_json,
+        within_amp_path=args.within_amp_json,
+        required_mode=args.required_mode,
+    ).to_dict()
+
+    if args.out:
+        Path(args.out).write_text(json.dumps(verdict, indent=2) + "\n")
+    print(json.dumps(verdict, indent=2))
+    return 0 if verdict["may_use_weighted_mode"] else 3

src/openamp_foundry/cli/commands/reports.py

@@ -810,8 +810,6 @@ def _run_recalibration_engine(args: argparse.Namespace) -> int:
         BudgetExceededError,
         PolicyViolationError,
         compute_weight_update,
-        write_weight_update_proposal_json,
-        write_weight_update_proposal_markdown,
     )
     from openamp_foundry.calibration.recalibration_gate import GateVerdict
 
@@ -899,7 +897,7 @@ def _run_recalibration_engine(args: argparse.Namespace) -> int:
         notes = prop_section.get("notes", [])
         print(f"  Notes: {', '.join(notes) if notes else '(none)'}")
         print(f"{'=' * 60}")
-        print(f"DRY RUN — no files written, no weight changes applied.")
+        print("DRY RUN — no files written, no weight changes applied.")
     else:
         if args.out_json:
             write_recalibration_report_json(report, args.out_json)