diff --git a/Makefile b/Makefile index 07b04eed..81f315bb 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-predict pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -41,6 +41,9 @@ help: @echo " make bench-order-dependent Analyze which features survive scrambling (order-dependence)" @echo " make bench-precision-at-k Precision@k operating characteristic (small-k triage, threshold calibration)" @echo " make bench-per-family Per-family benchmark — stratify 500 AMPs by structural class, per-class AUROC" + @echo " make bench-simulation-ablation Simulation ablation on AMP-vs-decoy benchmark" + @echo " make bench-simulation-ablation-within-amp Simulation ablation on hemolytic vs selective AMPs" + @echo " make simulation-gate Fail-closed gate for weighted simulation mode" @echo " make bench-gate Benchmark regression gate (AUROC drift check)" @echo " make regenerate-all Run all pipeline + benchmarks and verify determinism" @echo " make bench-hidden-active Hidden-positive recovery on mixed benchmark set" @@ -445,6 +448,12 @@ bench-simulation-ablation-within-amp: PYTHONPATH=src $(PYTHON) scripts/benchmark_simulation_ablation.py \ --mode within-amp --out outputs/simulation_ablation_within_amp.json +simulation-gate: + PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench simulation-gate \ + --amp-vs-decoy-json outputs/simulation_ablation.json \ + --within-amp-json outputs/simulation_ablation_within_amp.json \ + --out outputs/simulation_gate_verdict.json + bench-cross-dataset: PYTHONPATH=src $(PYTHON) scripts/benchmark_cross_dataset.py \ --out outputs/cross_dataset_benchmark.json diff --git a/docs/50_LOOP_PLAN.md b/docs/50_LOOP_PLAN.md index 3787e5c7..c8172cba 100644 --- a/docs/50_LOOP_PLAN.md +++ b/docs/50_LOOP_PLAN.md @@ -1,7 +1,7 @@ # 50-Loop Execution Plan -> **Status:** Strategic roadmap. Updated v0.5.50. -> **Current state:** 1843 tests, pipeline AUROC 0.7792, calibration pipeline complete (intake + gate + engine + dry-run + policy-version + synthetic generator + recalibration report + batch-2 selector + recovery benchmark + end-to-end integration + pytest golden-path test + policy bump workflow + negative result archive), +> **Status:** Strategic roadmap. Updated v0.5.56. +> **Current state:** 1927 passing tests, pipeline AUROC 0.7792, calibration pipeline complete (intake + gate + engine + dry-run + policy-version + synthetic generator + recalibration report + batch-2 selector + recovery benchmark + end-to-end integration + pytest golden-path test + policy bump workflow + negative result archive), > > Phase 0 complete (Loops 1–8), Phase 1 complete (Loops 9–17), Phase 2 complete (Loops 18–29), > cluster-split/selectivity/triage now gated in CI, Wave 0.5 panel ready (24 candidates, 15 families), no wet-lab data yet. > @@ -103,8 +103,8 @@ Build the multi-resolution virtual assay layer: structure proxies, membrane inte | 32 ✅ | No structure ensemble proxy. Current pipeline assumes all AMPs are helical | `simulation/structure.py`: StructureProxy with Chou-Fasman 3-state propensities, normalized ensemble weights, non_helical flag. v0.5.53 | 34 tests, 1905 total. Protegrin/tachyplesin non-helical (✅), magainin not non-helical (✅). Melittin ambiguous — Chou-Fasman limitation documented | | 33 ✅ | No per-module ablation: do simulation modules improve discrimination? (Original 'selectivity ratio' superseded by Loop 31 membrane proxy — replaced with more useful ablation benchmark) | `scripts/benchmark_simulation_ablation.py`: tests MembraneProxy + StructureProxy on 500-AMP AMP-vs-decoy benchmark. Reports per-module AUROC, combined composite AUROC, and verdict. v0.5.54 | 17 tests, 1922 total. Honest finding: composite degrades AMP-vs-decoy (delta=-0.1153), but bacterial_binding alone achieves 0.7512 AUROC — genuine non-charge signal from Wimley-White + hydrophobic moment | | 34 ✅ | No within-AMP simulation ablation: do simulation modules improve hemolysis detection? (Original 'membrane reference CSV' deferred — higher leverage to test modules on their actual task first) | Extended `scripts/benchmark_simulation_ablation.py` with `--mode within-amp`. Tests MembraneProxy + StructureProxy on 45 hemolytic vs 125 selective AMPs. Reports per-score detection AUROC. v0.5.55 | 23 tests, 1928 total. Honest finding: best simulation helix_weight AUROC 0.6458; rich_selectivity still best at 0.7453. Simulation modules do NOT improve over existing scorers for hemolysis detection | -| 35 | No per-module ablation: each simulation module must beat a cheap heuristic baseline before affecting selection | `benchmark/simulation_ablation.py`: compares each simulation module against trivial baseline (single-feature proxy). Reports delta AUROC for each | Any module with delta ≤ 0 is flagged as `NO_IMPROVEMENT` and disabled from production pipeline | -| 36 | No uncertainty-aware composite that folds simulation outputs into the ranking | `scoring/simulation_composite.py`: weighted combination of simulation modules, weighted by their benchmarked AUROC. Uncertainty from `SimulationResult` propagates to final score | Uncertainty reported alongside every candidate score in evidence certificate | +| 35 ✅ | No executable brake prevented future agents from enabling weighted simulation despite negative ablation results | `simulation/gate.py` + `openamp-foundry bench simulation-gate`: fail-closed permission gate consuming both current ablation artifacts. `make simulation-gate` writes a machine-readable verdict. | 6 tests added; current suite 1927 passing, 7 skipped. Current verdict blocks weighted mode and downgrades to info-only | +| 36 | No per-module cheap-baseline benchmark for within-domain simulation value | `benchmark/simulation_baselines.py`: compare each simulation signal against its cheap heuristic baseline on the same task. Report deltas + CI. | Any module with delta ≤ 0 remains permanently experimental | | 37 | No simulation module that meets the bar to affect selection. All may be experimental | Add `--simulation-mode` flag to the `rank` command: `off` (default, current pipeline), `info` (simulation scores in report only, no ranking impact), `weighted` (simulation affects ranking if benchmarked AUROC > 0.6) | Each mode produces distinct, documented output | | 38 | No API contract for third-party simulation modules (e.g. Martini MD, AlphaFold) | `simulation/interfaces.py` extended with `ExternalSimulationAdapter` protocol. Documented in ARCHITECTURE.md extension points | Third-party can implement the protocol without reading pipeline internals | | 39 | Virtual-assay benchmark report comparing simulation-augmented vs simulation-free ranking on the strict triage set | `docs/SIMULATION_BENCHMARK.md`: head-to-head comparison of sel_vs_dec, hem_vs_dec, sel_vs_hem AUROCs with and without simulation. If simulation doesn't improve strict triage, say so explicitly | Honest: if delta ≤ 0, report "simulation did not improve triage" | @@ -180,7 +180,7 @@ If no data arrives, virtual assay scaffolding continues independently. Phase 0: ✅ Complete (Loops 1–8) Phase 1: ✅ Complete (Loops 9–17) Phase 2: ✅ Complete (Loops 18–29) -Phase 3: In progress — Loop 34 ✅ (Loops 30–39) +Phase 3: In progress — Loop 35 ✅ (Loops 30–39) Phase 4: Not started (Loops 40–49) ``` @@ -241,7 +241,7 @@ Phase 4: Not started (Loops 40–49) | 28 ✅ | Policy version bump workflow for when real data arrives | `scripts/bump_recalibration_policy.py`: standalone script with `--dry-run`, decision-log guard, auto-increment + write. CI guard in `ci.yml` validates policy version changes against base branch. v0.5.49. 9 tests. | CI rejects policy PRs without valid decision log; `make bump-policy-version` and `make bump-policy-version-dry-run` available | | 29 ✅ | No public negative-result archive format. If Wave 1 yields all negatives, where do they go? | `docs/NEGATIVE_RESULT_ARCHIVE.md`: full template with entry schema, procedures, automation notes, and limitations. Covers pre-selection rejects, selected-untested, lab inactives, lab toxic, control failures. v0.5.50. | Template complete enough for a lab partner to fill; schema defines 18 fields with required/conditional markers | -**Next loop:** Loop 35 — Phase 3 (membrane reference benchmark set or simulation-mode CLI flag). +**Next loop:** Loop 36 — Phase 3 cheap-baseline benchmark for each simulation signal. **Phase 2 exit criteria (all 5 met ✅):** - ✅ `make calibration-loop` runs from clean checkout, produces batch-2 manifest diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index c9794532..82c7a9ad 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -73,7 +73,7 @@ documented result in `docs/METRICS_CURRENT.md` and | Benchmark | Purpose | Reason deferred | |-----------|---------|----------------| | Time split | Test future generalization | Metadata not available for most sequences | -| Virtual-assay ablation | Test whether simulation modules improve triage | Phase 3 — no modules exist yet | +| Virtual-assay ablation | Test whether simulation modules improve triage | Shipped: `make bench-simulation-ablation`, `make bench-simulation-ablation-within-amp`, `make simulation-gate` | | Biologically plausible charge-balanced negative set | Honest non-charge signal test | Requires curated negative sequences — see v0.5.41 recommendation | ## Minimum report fields diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index 311ce607..cec10a35 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -5,7 +5,12 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-06 (within-AMP simulation ablation — v0.5.55) +> **Last updated:** 2026-07-06 (simulation weighted-mode gate — v0.5.56) +> **New in v0.5.56:** `openamp-foundry bench simulation-gate` converts both +> simulation ablation artifacts into a fail-closed permission decision for +> `weighted` mode. Current verdict: blocked. AMP-vs-decoy delta remains +> negative and within-AMP delta remains negative, so simulation stays +> informational only. 6 tests. Current suite: 1927 passed, 7 skipped. > **New in v0.5.55:** Within-AMP simulation ablation — `--mode within-amp` tests MembraneProxy + StructureProxy on hemolysis detection (45 hemolytic vs 125 selective). Honest finding: best simulation `helix_weight` AUROC 0.6458; `rich_selectivity` still best at 0.7453. 23 tests. 1928 total. > **New in v0.5.54:** Simulation ablation benchmark — tests MembraneProxy + StructureProxy on 500-AMP AMP-vs-decoy benchmark. Honest finding: composite degrades (delta=−0.1153) but `bacterial_binding` alone achieves AUROC 0.7512 — genuine non-charge signal. 17 new tests. 1922 total. > **New in v0.5.53:** Structure ensemble proxy — `StructureProxy` using Chou-Fasman 3-state helix/sheet/coil propensities. `non_helical` flag for helic-biased scorer warnings. `HelicityBaseline`. 34 new tests. 1905 total. @@ -36,7 +41,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.55 +> **Pipeline version:** v0.5.56 > **Branch:** main --- @@ -1106,7 +1111,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 1928 | +| Total tests | 1927 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 4420162d..e542644f 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,5 +1,33 @@ # Roadmap +## v0.5.56 — Simulation Weighted-Mode Gate (Loop 35) ✓ (2026-07-06) + +Phase 3 had a real honesty gap: the scope doc described `weighted` simulation +mode, but no executable guard converted ablation evidence into an allow/block +decision. That left future agents one edit away from integration theater. + +Changes: +- ``src/openamp_foundry/simulation/gate.py`` — fail-closed gate that consumes + current AMP-vs-decoy and within-AMP ablation artifacts and returns a + machine-readable permission verdict for simulation integration mode. +- ``openamp-foundry bench simulation-gate`` — CLI wrapper with exit code `0` + only when weighted mode is honestly allowed, `3` otherwise. +- ``Makefile`` — ``make simulation-gate`` target writing + ``outputs/simulation_gate_verdict.json`` from both ablation artifacts. +- ``docs/VIRTUAL_ASSAY_SCOPE.md`` — corrected stale claim that `rank` already + accepts `--simulation-mode`; current state now says weighted mode is blocked + until the gate passes. +- ``tests/test_simulation_gate.py`` — 6 tests covering blocked current-state + verdicts, positive-path permission, fail-closed missing artifacts, and CLI + exit behavior. +- Full verification after this loop: ``1927 passed, 7 skipped``. + +**Honest finding:** +- Weighted simulation remains blocked. Current ablations do not justify ranking + impact. +- This is not a modeling advance. It is a truthfulness and architecture + advance: the repo now has an executable brake against simulation theater. + ## v0.5.55 — Within-AMP Simulation Ablation (Loop 34) ✓ (2026-07-06) Extended the simulation ablation benchmark to test modules on their diff --git a/docs/VIRTUAL_ASSAY_SCOPE.md b/docs/VIRTUAL_ASSAY_SCOPE.md index 1b327cec..68f4fbdd 100644 --- a/docs/VIRTUAL_ASSAY_SCOPE.md +++ b/docs/VIRTUAL_ASSAY_SCOPE.md @@ -9,9 +9,15 @@ an impressive-looking model that does not improve decision quality is noise. ## Status -**Phase 3 — Planning.** No virtual-assay modules are implemented in the -production pipeline. This document describes the intended scope and -constraints. Implementation starts with Loop 31. +**Phase 3 — experimental modules implemented, production integration still gated.** +`MembraneProxy` (Loop 31) and `StructureProxy` (Loop 32) exist, and both now +have explicit ablation evidence. Current state: + +- `bench-simulation-ablation` shows simulation composite degrades AMP-vs-decoy. +- `bench-simulation-ablation-within-amp` shows simulation scores do not beat + `rich_selectivity` on hemolysis detection. +- `bench simulation-gate` therefore blocks `weighted` mode and downgrades to + `info` until the evidence changes. ## Core Principle @@ -88,9 +94,12 @@ The virtual assay layer supports three modes: |------|--------|-------------| | `off` (default) | No simulation scores computed. Current pipeline. | Production candidate ranking | | `info` | Simulation scores computed and included in reports only. No ranking impact. | Benchmarking, exploration | -| `weighted` | Simulation scores affect ranking if each module clears its ablation bar. | Only after ablation validation | +| `weighted` | Simulation scores affect ranking only if the simulation gate passes on current ablation artifacts. | Only after ablation validation | -The `rank` CLI accepts `--simulation-mode` to select the mode. +The `rank` CLI does **not** yet expose `--simulation-mode`. That omission is +intentional until a module clears the gate honestly. Current executable policy: +`openamp-foundry bench simulation-gate` may return `weighted` permission in the +future, but production ranking remains simulation-free today. ## Calibration Requirement diff --git a/src/openamp_foundry/active_learning/benchmark.py b/src/openamp_foundry/active_learning/benchmark.py index c301bcd5..63d9912b 100644 --- a/src/openamp_foundry/active_learning/benchmark.py +++ b/src/openamp_foundry/active_learning/benchmark.py @@ -22,7 +22,7 @@ import csv import logging import random -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path from typing import Any diff --git a/src/openamp_foundry/active_learning/selector.py b/src/openamp_foundry/active_learning/selector.py index af603232..acbeb896 100644 --- a/src/openamp_foundry/active_learning/selector.py +++ b/src/openamp_foundry/active_learning/selector.py @@ -9,7 +9,7 @@ import csv import logging -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path _log = logging.getLogger(__name__) diff --git a/src/openamp_foundry/cli/commands/benchmark.py b/src/openamp_foundry/cli/commands/benchmark.py index 4e1c5c3a..d3cf5130 100644 --- a/src/openamp_foundry/cli/commands/benchmark.py +++ b/src/openamp_foundry/cli/commands/benchmark.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse import json +from pathlib import Path def _run_bench(args: argparse.Namespace) -> int: from openamp_foundry.data.loaders import load_candidates_csv @@ -320,3 +321,19 @@ def _run_active_learning_bench(args: argparse.Namespace) -> int: out_dict = result.to_dict() print(json.dumps(out_dict, indent=2)) return 0 + + +def _run_simulation_gate(args: argparse.Namespace) -> int: + """Validate whether virtual-assay scores may affect ranking.""" + from openamp_foundry.simulation.gate import evaluate_simulation_gate + + verdict = evaluate_simulation_gate( + amp_vs_decoy_path=args.amp_vs_decoy_json, + within_amp_path=args.within_amp_json, + required_mode=args.required_mode, + ).to_dict() + + if args.out: + Path(args.out).write_text(json.dumps(verdict, indent=2) + "\n") + print(json.dumps(verdict, indent=2)) + return 0 if verdict["may_use_weighted_mode"] else 3 diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 9426de5e..adb834d1 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -810,8 +810,6 @@ def _run_recalibration_engine(args: argparse.Namespace) -> int: BudgetExceededError, PolicyViolationError, compute_weight_update, - write_weight_update_proposal_json, - write_weight_update_proposal_markdown, ) from openamp_foundry.calibration.recalibration_gate import GateVerdict @@ -899,7 +897,7 @@ def _run_recalibration_engine(args: argparse.Namespace) -> int: notes = prop_section.get("notes", []) print(f" Notes: {', '.join(notes) if notes else '(none)'}") print(f"{'=' * 60}") - print(f"DRY RUN — no files written, no weight changes applied.") + print("DRY RUN — no files written, no weight changes applied.") else: if args.out_json: write_recalibration_report_json(report, args.out_json) diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index f395b9c9..f16138f4 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -1,7 +1,7 @@ from __future__ import annotations from openamp_foundry.cli.commands.core import _run_generate_batch -from openamp_foundry.cli.commands.benchmark import _run_bench, _run_validate_scoring, _run_cluster_split_bench, _run_expert_ablation_bench, _run_selectivity_bench, _run_triage, _run_metrics_snapshot, _run_feature_decomp, _run_active_learning_bench +from openamp_foundry.cli.commands.benchmark import _run_bench, _run_validate_scoring, _run_cluster_split_bench, _run_expert_ablation_bench, _run_selectivity_bench, _run_triage, _run_metrics_snapshot, _run_feature_decomp, _run_active_learning_bench, _run_simulation_gate from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc @@ -296,6 +296,36 @@ def build_parser() -> argparse.ArgumentParser: help="Random seed for reproducibility (default: 42).", ) + simulation_gate = bench_sub.add_parser( + "simulation-gate", + help=( + "Validate whether virtual-assay weighted mode is allowed. " + "Reads ablation artifacts and fails closed unless simulations " + "improve on both benchmark views." + ), + ) + simulation_gate.add_argument( + "--amp-vs-decoy-json", + required=True, + help="Ablation JSON from scripts/benchmark_simulation_ablation.py --mode amp-vs-decoy.", + ) + simulation_gate.add_argument( + "--within-amp-json", + required=True, + help="Ablation JSON from scripts/benchmark_simulation_ablation.py --mode within-amp.", + ) + simulation_gate.add_argument( + "--required-mode", + choices=["off", "info", "weighted"], + default="weighted", + help="Requested simulation integration mode (default: weighted).", + ) + simulation_gate.add_argument( + "--out", + required=False, + help="Optional JSON output path.", + ) + generate = sub.add_parser( "generate-batch", help=( @@ -1046,6 +1076,8 @@ def main(argv: list[str] | None = None) -> int: return _run_metrics_snapshot(args) if args.bench_command == "active-learning": return _run_active_learning_bench(args) + if args.bench_command == "simulation-gate": + return _run_simulation_gate(args) return _run_bench(args) if args.command == "generate-batch": diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index 7377c4f0..fe391943 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -1,5 +1,6 @@ """Virtual assay simulation layer for OpenAMP Foundry.""" +from .gate import SimulationGateVerdict, evaluate_simulation_gate from .interfaces import ( EmulatorBaseline, SimulationResult, @@ -14,4 +15,6 @@ "EmulatorBaseline", "MembraneProxy", "StructureProxy", + "SimulationGateVerdict", + "evaluate_simulation_gate", ] diff --git a/src/openamp_foundry/simulation/gate.py b/src/openamp_foundry/simulation/gate.py new file mode 100644 index 00000000..4306643d --- /dev/null +++ b/src/openamp_foundry/simulation/gate.py @@ -0,0 +1,126 @@ +"""Validation gate for virtual-assay weighted mode. + +The gate converts ablation benchmark outputs into an explicit permission +decision. Weighted simulation must fail closed until both current ablation +views show improvement over existing cheap/pipeline scorers. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +MIN_WITHIN_AMP_DELTA = 0.03 +MIN_AMP_VS_DECOY_DELTA = 0.005 + + +@dataclass(frozen=True) +class SimulationGateVerdict: + """Machine-readable permission decision for simulation ranking modes.""" + + may_use_weighted_mode: bool + integration_mode: str + required_mode: str + checks: dict[str, bool] + reasons: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "may_use_weighted_mode": self.may_use_weighted_mode, + "integration_mode": self.integration_mode, + "required_mode": self.required_mode, + "checks": self.checks, + "reasons": self.reasons, + } + + +def _load_json(path: str | Path) -> dict[str, Any]: + p = Path(path) + if not p.exists(): + raise FileNotFoundError(f"simulation ablation artifact not found: {p}") + payload = json.loads(p.read_text()) + if not isinstance(payload, dict): + raise ValueError(f"simulation ablation artifact must be a JSON object: {p}") + return payload + + +def _result(payload: dict[str, Any], key: str, default: Any = None) -> Any: + results = payload.get("results", {}) + if not isinstance(results, dict): + return default + return results.get(key, default) + + +def evaluate_simulation_gate( + *, + amp_vs_decoy_path: str | Path, + within_amp_path: str | Path, + required_mode: str = "weighted", +) -> SimulationGateVerdict: + """Evaluate whether simulation outputs may affect candidate selection. + + Confirmed policy: + - `off` and `info` are always allowed because they do not affect ranking. + - `weighted` requires both ablation artifacts and positive improvement. + - Missing or malformed artifacts raise, rather than silently passing. + """ + + if required_mode not in {"off", "info", "weighted"}: + raise ValueError("required_mode must be one of: off, info, weighted") + + if required_mode in {"off", "info"}: + return SimulationGateVerdict( + may_use_weighted_mode=False, + integration_mode=required_mode, + required_mode=required_mode, + checks={ + "weighted_requested": False, + "amp_vs_decoy_improves": False, + "within_amp_improves": False, + }, + reasons=[ + f"{required_mode} mode does not affect ranking; ablation permission not required." + ], + ) + + amp_vs_decoy = _load_json(amp_vs_decoy_path) + within_amp = _load_json(within_amp_path) + + amp_delta = float(_result(amp_vs_decoy, "delta", 0.0)) + within_delta = float(_result(within_amp, "delta", 0.0)) + amp_verdict = _result(amp_vs_decoy, "verdict", "UNKNOWN") + within_verdict = _result(within_amp, "verdict", "UNKNOWN") + + checks = { + "weighted_requested": True, + "amp_vs_decoy_improves": amp_verdict == "IMPROVEMENT" + and amp_delta > MIN_AMP_VS_DECOY_DELTA, + "within_amp_improves": within_verdict == "IMPROVEMENT" + and within_delta > MIN_WITHIN_AMP_DELTA, + } + + reasons: list[str] = [] + if not checks["amp_vs_decoy_improves"]: + reasons.append( + "AMP-vs-decoy ablation does not improve over ensemble " + f"(verdict={amp_verdict}, delta={amp_delta:+.4f})." + ) + if not checks["within_amp_improves"]: + reasons.append( + "Within-AMP ablation does not improve over existing scorers " + f"(verdict={within_verdict}, delta={within_delta:+.4f})." + ) + + may_use_weighted = all(checks.values()) + return SimulationGateVerdict( + may_use_weighted_mode=may_use_weighted, + integration_mode="weighted" if may_use_weighted else "info", + required_mode=required_mode, + checks=checks, + reasons=reasons + if reasons + else ["Both simulation ablation views clear the weighted-mode bar."], + ) diff --git a/tests/test_benchmark_simulation_ablation.py b/tests/test_benchmark_simulation_ablation.py index d2ae6aec..a3dfd5bb 100644 --- a/tests/test_benchmark_simulation_ablation.py +++ b/tests/test_benchmark_simulation_ablation.py @@ -4,8 +4,6 @@ import subprocess import sys -import pytest - from openamp_foundry.simulation.membrane import MembraneProxy from openamp_foundry.simulation.structure import StructureProxy from scripts.benchmark_simulation_ablation import ( @@ -141,7 +139,7 @@ def test_cli_within_amp_exit_code(): def test_cli_writes_output_json(tmp_path): out = tmp_path / "ablation.json" - result = subprocess.run( + subprocess.run( [sys.executable, "scripts/benchmark_simulation_ablation.py", "--mode", "amp-vs-decoy", "--out", str(out)], capture_output=True, text=True, diff --git a/tests/test_bump_recalibration_policy.py b/tests/test_bump_recalibration_policy.py index 53232dce..5f786f82 100644 --- a/tests/test_bump_recalibration_policy.py +++ b/tests/test_bump_recalibration_policy.py @@ -6,12 +6,8 @@ import os import subprocess import sys -import tempfile from pathlib import Path -import pytest -import yaml - from openamp_foundry.calibration.policy import load_recalibration_policy MINIMAL_POLICY = """\ @@ -327,4 +323,3 @@ def test_bumpable_twice(self, tmp_path: Path) -> None: assert loaded_2.policy_version == 3 assert loaded_2.human_reviewer == "r2" assert loaded_2.locked_at == "2026-08-02" - diff --git a/tests/test_calibration_e2e.py b/tests/test_calibration_e2e.py index 9d2ef4ac..bc05e135 100644 --- a/tests/test_calibration_e2e.py +++ b/tests/test_calibration_e2e.py @@ -1235,7 +1235,6 @@ def test_full_calibration_loop_via_cli(self, tmp_path): # Reconstruct GateVerdict from the gate output from openamp_foundry.calibration.recalibration_gate import ( - GateVerdict, evaluate_recalibration_gate, ) diff --git a/tests/test_policy_version.py b/tests/test_policy_version.py index a841fd46..e9632592 100644 --- a/tests/test_policy_version.py +++ b/tests/test_policy_version.py @@ -2,22 +2,16 @@ from __future__ import annotations -import json -import os import tempfile -from datetime import date, timedelta +from datetime import date from pathlib import Path -import pytest import yaml from openamp_foundry.calibration.policy import ( LockedChange, PolicyRule, - ProhibitedAction, - RateLimit, RecalibrationPolicy, - ReviewerArtefact, ) from openamp_foundry.calibration.policy_version import ( VersionValidation, diff --git a/tests/test_simulation_gate.py b/tests/test_simulation_gate.py new file mode 100644 index 00000000..e08094c5 --- /dev/null +++ b/tests/test_simulation_gate.py @@ -0,0 +1,137 @@ +"""Tests for the simulation weighted-mode permission gate.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from openamp_foundry.simulation.gate import ( + MIN_AMP_VS_DECOY_DELTA, + MIN_WITHIN_AMP_DELTA, + evaluate_simulation_gate, +) + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload, indent=2) + "\n") + + +def test_simulation_gate_blocks_current_ablation_results(tmp_path: Path) -> None: + amp = tmp_path / "amp.json" + within = tmp_path / "within.json" + _write_json(amp, {"results": {"verdict": "NO_IMPROVEMENT", "delta": -0.1153}}) + _write_json(within, {"results": {"verdict": "NO_IMPROVEMENT", "delta": -0.0995}}) + + verdict = evaluate_simulation_gate( + amp_vs_decoy_path=amp, + within_amp_path=within, + ) + + assert verdict.may_use_weighted_mode is False + assert verdict.integration_mode == "info" + assert verdict.checks["amp_vs_decoy_improves"] is False + assert verdict.checks["within_amp_improves"] is False + assert len(verdict.reasons) == 2 + + +def test_simulation_gate_allows_weighted_only_on_double_improvement(tmp_path: Path) -> None: + amp = tmp_path / "amp.json" + within = tmp_path / "within.json" + _write_json( + amp, + {"results": {"verdict": "IMPROVEMENT", "delta": MIN_AMP_VS_DECOY_DELTA + 0.0100}}, + ) + _write_json( + within, + {"results": {"verdict": "IMPROVEMENT", "delta": MIN_WITHIN_AMP_DELTA + 0.0100}}, + ) + + verdict = evaluate_simulation_gate( + amp_vs_decoy_path=amp, + within_amp_path=within, + ) + + assert verdict.may_use_weighted_mode is True + assert verdict.integration_mode == "weighted" + assert all(verdict.checks.values()) + + +def test_simulation_gate_info_mode_never_requires_ablation(tmp_path: Path) -> None: + verdict = evaluate_simulation_gate( + amp_vs_decoy_path=tmp_path / "missing.json", + within_amp_path=tmp_path / "missing2.json", + required_mode="info", + ) + + assert verdict.may_use_weighted_mode is False + assert verdict.integration_mode == "info" + assert "does not affect ranking" in verdict.reasons[0] + + +def test_simulation_gate_missing_file_raises(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + evaluate_simulation_gate( + amp_vs_decoy_path=tmp_path / "missing.json", + within_amp_path=tmp_path / "missing2.json", + ) + + +def test_cli_simulation_gate_blocks(tmp_path: Path) -> None: + amp = tmp_path / "amp.json" + within = tmp_path / "within.json" + out = tmp_path / "verdict.json" + _write_json(amp, {"results": {"verdict": "NO_IMPROVEMENT", "delta": -0.10}}) + _write_json(within, {"results": {"verdict": "NO_IMPROVEMENT", "delta": -0.10}}) + + result = subprocess.run( + [ + sys.executable, + "-m", + "openamp_foundry.cli", + "bench", + "simulation-gate", + "--amp-vs-decoy-json", + str(amp), + "--within-amp-json", + str(within), + "--out", + str(out), + ], + capture_output=True, + text=True, + env={"PYTHONPATH": "src"}, + ) + + assert result.returncode == 3 + payload = json.loads(out.read_text()) + assert payload["may_use_weighted_mode"] is False + + +def test_cli_simulation_gate_allows(tmp_path: Path) -> None: + amp = tmp_path / "amp.json" + within = tmp_path / "within.json" + _write_json(amp, {"results": {"verdict": "IMPROVEMENT", "delta": 0.0200}}) + _write_json(within, {"results": {"verdict": "IMPROVEMENT", "delta": 0.0500}}) + + result = subprocess.run( + [ + sys.executable, + "-m", + "openamp_foundry.cli", + "bench", + "simulation-gate", + "--amp-vs-decoy-json", + str(amp), + "--within-amp-json", + str(within), + ], + capture_output=True, + text=True, + env={"PYTHONPATH": "src"}, + ) + + assert result.returncode == 0