diff --git a/Makefile b/Makefile index 67c93222..2011f939 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -756,3 +756,8 @@ simulation-ci-report: .venv/bin/openamp-foundry simulation-ci-report \ --results-json '[{"module":"membrane_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.75},"uncertainty":0.1,"calibration_set":null,"validated_against":["APD3"],"notes":[]},{"module":"structure_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.80},"uncertainty":0.15,"calibration_set":null,"validated_against":["APD3"],"notes":[]}]' @echo "CI report complete." + +simulation-deprecation-check: + .venv/bin/openamp-foundry simulation-deprecation-check \ + --module-ids membrane_proxy,dummy_membrane_proxy,external_adapter_placeholder + @echo "Deprecation check complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 40b0a659..eab77ece 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase H H7 — simulation-result confidence interval reporter — v0.5.95) +> **Last updated:** 2026-07-09 (Phase H H8 — simulation-module deprecation enforcer — v0.5.96) +> **New in v0.5.96:** Simulation-module deprecation enforcer (H8) — `DeprecationCheckResult` dataclass (5 fields: module_id, status, is_blocked, block_reason, dry_lab_only). `BLOCKED_STATUSES = {"deprecated", "unavailable"}`. `check_module_deprecation()` looks up module in registry: not-found returns is_blocked=True status="unknown"; deprecated/unavailable returns is_blocked=True with reason; active/experimental returns is_blocked=False. `enforce_deprecation()` filters list[SimulationResult] to only non-blocked modules, returns total_input/passed/blocked/blocked_modules/passed_results/checks/dry_lab_only. `run_deprecation_check_batch()` bulk-checks module_ids with total/blocked/allowed/any_blocked/results/dry_lab_only. CLI (`openamp-foundry simulation-deprecation-check`) with `--module-ids`, `--format text|json`. `make simulation-deprecation-check` target. 21 tests. **3238 total.** Deprecated simulation modules must not be used in production scoring — the enforcer prevents stale or unreliable modules from tainting evidence packets. > **New in v0.5.95:** Simulation-result confidence interval reporter (H7) — `ci_reporter.py` with `ScoreCI` dataclass (9 fields: module_id, score_key, point_estimate, uncertainty, ci_lower, ci_upper, ci_width, overlaps_with, dry_lab_only). `compute_score_ci()` builds CIs from SimulationResult.scores[score_key] ± uncertainty, returns None if score_key missing. `compare_cis()` pairwise checks overlap condition (a_lo <= b_hi and b_lo <= a_hi), returns new list with overlaps_with populated (no in-place mutation). `ci_report()` produces full report with n_results, cis list, any_overlap flag, and dry_lab_only=True. CLI (`openamp-foundry simulation-ci-report`) with `--results-json`, `--score-key`, `--format text|json`. `make simulation-ci-report` target. 16 tests. **3217 total.** Raw scores without uncertainty ranges make it impossible to judge whether two candidates are distinguishable. The CI reporter makes uncertainty explicit and auditable. > **New in v0.5.94:** Simulation-ensemble agreement checker (H6) — `ensemble_checker.py` with `EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked, agreement_level, agreement_description, mean_score, score_range, scores_by_module, threshold, dry_lab_only). `AGREEMENT_LEVELS` dict (5 levels: strong, moderate, weak, conflict, insufficient). `check_ensemble_agreement()` extracts score_key from each SimulationResult's scores dict, computes score_range, classifies into strong (≥3 modules within threshold), moderate (2 modules within threshold), weak (1 module), conflict (beyond threshold), or insufficient (no results). `run_ensemble_check_batch()` aggregates multiple calls with counts and any_conflict flag. CLI (`openamp-foundry simulation-ensemble-check`) with `--sequence`, `--results-json`, `--score-key`, `--threshold`, `--format text|json`. `make simulation-ensemble-check` target. 20 tests. **3201 total.** When multiple simulation modules independently agree on a candidate, that agreement is stronger evidence than a single module alone. The ensemble checker makes this agreement explicit and auditable. > **New in v0.5.93:** Simulation-result provenance chain (H5) — `SimulationProvenanceRecord` dataclass with run_id, module_id, module_version, timestamp_utc, input_hash (SHA-256 of input sequence), result_hash (SHA-256 of sorted scores dict), calibration_set, notes, and dry_lab_only. `make_provenance_record()` computes input_hash and result_hash deterministically (sort_keys=True for result_hash). `validate_provenance_record()` checks run_id, module_id, module_version non-empty; ISO 8601 timestamp contains 'T'; hashes are 64-char hex; dry_lab_only is True. `provenance_summary()` aggregates total, unique modules, run_ids, and dry_lab_only flag. CLI (`openamp-foundry simulation-provenance`) with `--run-id`, `--module-id`, `--module-version`, `--timestamp-utc`, `--input-sequence`, `--scores-json`, `--calibration-set`, `--format text|json`. `make simulation-provenance` target with demo invocation. 19 tests. **3181 total.** Every simulation result carries a traceable provenance chain so results can be audited, reproduced, or invalidated later without relying on memory. @@ -86,7 +87,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.95 (H7 — simulation-result confidence interval reporter) +> **Pipeline version:** v0.5.96 (H8 — simulation-module deprecation enforcer) > **Branch:** main --- @@ -1156,7 +1157,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 3201 | +| Total tests | 3238 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index c119526f..4a16ee56 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -51,6 +51,52 @@ Honest boundaries: not invalidate results; it highlights them for human review. - All CI reports are dry-lab only and must not be presented as biological proof. +## v0.5.96 — Loop 96: Phase H H8 — Simulation-Module Deprecation Enforcer ✓ (2026-07-09) + +`DeprecationCheckResult` dataclass (5 fields: module_id, status, is_blocked, +block_reason, dry_lab_only). `BLOCKED_STATUSES = {"deprecated", "unavailable"}`. +`check_module_deprecation()` looks up module in registry: not-found returns +is_blocked=True status="unknown"; deprecated/unavailable returns is_blocked=True +with reason; active/experimental returns is_blocked=False. +`enforce_deprecation()` filters list[SimulationResult] to only non-blocked +modules, returns dict with total_input/passed/blocked/blocked_modules/ +passed_results/checks/dry_lab_only. `run_deprecation_check_batch()` bulk-checks +module_ids with total/blocked/allowed/any_blocked/results/dry_lab_only. +CLI (`openamp-foundry simulation-deprecation-check`) with `--module-ids`, +`--format text|json`. `make simulation-deprecation-check` target with demo +invocation. Deprecated simulation modules must not be used in production +scoring — the enforcer prevents stale or unreliable modules from tainting +evidence packets. + +Changes: +- `src/openamp_foundry/simulation/deprecation_enforcer.py` (H8) — Core + module with `DeprecationCheckResult` dataclass, `check_module_deprecation()`, + `enforce_deprecation()`, `run_deprecation_check_batch()`. +- `src/openamp_foundry/simulation/__init__.py` — Exports `BLOCKED_STATUSES`, + `DeprecationCheckResult`, `check_module_deprecation`, `enforce_deprecation`, + `run_deprecation_check_batch`. +- `src/openamp_foundry/cli/main.py` — Registered `simulation-deprecation-check` + subcommand with `--module-ids`, `--format` flags. Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_simulation_deprecation_check()` CLI handler. +- `Makefile` — Added `simulation-deprecation-check` target. Added to `.PHONY`. +- `tests/simulation/test_deprecation_enforcer.py` — 21 tests covering: active + module not blocked, deprecated/unavailable blocked, unknown blocked with + status="unknown", dry_lab_only always True, enforce_deprecation filters + results, passed count, blocked_modules sorted/deduplicated, dry_lab_only flag, + all-allowed blocked=0, batch total/any_blocked/not blocked. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.96 H8 changelog. Test count: 3238. +- `tests/test_test_count_regression.py` — baseline updated to 3238. + +Honest boundaries: +- Deprecation checks are based on registry status, not biological validity. + A module may be "active" in the registry but still produce unreliable scores. +- Blocking removes results from evidence packets but does not invalidate them + as research data — blocked results may still be useful for internal analysis. +- The enforcer does not evaluate whether a module's underlying science is sound; + it enforces the pipeline's declared module lifecycle policy. +- All outputs are dry-lab only and must not be presented as biological proof. + ## v0.5.94 — Loop 94: Phase H H6 — Simulation-Ensemble Agreement Checker ✓ (2026-07-09) `EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked, diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index be518788..542b27a0 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1704,3 +1704,40 @@ def _run_simulation_provenance(args: argparse.Namespace) -> int: print("not biological proof.") return 0 + + +def _run_simulation_deprecation_check(args: argparse.Namespace) -> int: + """Check whether simulation modules are deprecated or unavailable.""" + import json + + from openamp_foundry.simulation.deprecation_enforcer import ( + run_deprecation_check_batch, + ) + + module_ids = [m.strip() for m in args.module_ids.split(",") if m.strip()] + if not module_ids: + print(json.dumps({"status": "error", "error": "No module IDs provided."})) + return 2 + + result = run_deprecation_check_batch(module_ids) + output_format = getattr(args, "format", "text") + + if output_format == "json": + print(json.dumps(result, indent=2)) + else: + print(f"Simulation Deprecation Check") + print(f" Total modules: {result['total']}") + print(f" Blocked: {result['blocked']}") + print(f" Allowed: {result['allowed']}") + print(f" Any blocked: {result['any_blocked']}") + print() + for check in result["results"]: + status = "BLOCKED" if check["is_blocked"] else "ALLOWED" + print(f" [{status:7}] {check['module_id']:35s} " + f"status={check['status']:15s} " + f"{check['block_reason']}") + print() + print("Dry-lab only. Deprecation checks are computational safeguards,") + print("not biological proof.") + + return 3 if result["any_blocked"] else 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 43dbd378..76e8685c 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1519,6 +1519,25 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Simulation deprecation check (H8) ──────────────────────────── + sdc = sub.add_parser( + "simulation-deprecation-check", + help=( + "Check whether simulation modules are deprecated or unavailable. " + "Blocks deprecated modules from contributing to evidence packets. " + "Dry-lab only." + ), + ) + sdc.add_argument( + "--module-ids", type=str, required=True, + help="Comma-separated module IDs (e.g. membrane_proxy,dummy_membrane_proxy).", + ) + sdc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Simulation-result confidence interval report (H7) ──────────── sicr = sub.add_parser( "simulation-ci-report", @@ -1718,6 +1737,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "simulation-ci-report": return _run_simulation_ci_report(args) + if args.command == "simulation-deprecation-check": + return _run_simulation_deprecation_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index 9db970ab..0b48c3e2 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -1,5 +1,12 @@ """Virtual assay simulation layer for OpenAMP Foundry.""" +from .deprecation_enforcer import ( + BLOCKED_STATUSES, + DeprecationCheckResult, + check_module_deprecation, + enforce_deprecation, + run_deprecation_check_batch, +) from .adapter_gate import ( AdapterGateResult, FAIL_CLOSED_REASONS, @@ -56,6 +63,11 @@ from .structure import StructureProxy __all__ = [ + "BLOCKED_STATUSES", + "DeprecationCheckResult", + "check_module_deprecation", + "enforce_deprecation", + "run_deprecation_check_batch", "AdapterGateResult", "FAIL_CLOSED_REASONS", "evaluate_adapter_gate", diff --git a/src/openamp_foundry/simulation/deprecation_enforcer.py b/src/openamp_foundry/simulation/deprecation_enforcer.py new file mode 100644 index 00000000..cbf25b8c --- /dev/null +++ b/src/openamp_foundry/simulation/deprecation_enforcer.py @@ -0,0 +1,94 @@ +"""Deprecation enforcer for simulation modules — blocks deprecated or +unavailable modules from contributing to evidence packets. + +Dry-lab only. Blocking decisions are based on registry status, not +biological validity. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, asdict +from typing import Any + +from openamp_foundry.simulation.interfaces import SimulationResult +from openamp_foundry.simulation.module_registry import get_module_entry + +BLOCKED_STATUSES: set[str] = {"deprecated", "unavailable"} + + +@dataclass +class DeprecationCheckResult: + module_id: str + status: str + is_blocked: bool + block_reason: str + dry_lab_only: bool = True + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def check_module_deprecation(module_id: str) -> DeprecationCheckResult: + entry = get_module_entry(module_id) + if entry is None: + return DeprecationCheckResult( + module_id=module_id, + status="unknown", + is_blocked=True, + block_reason="Module not found in registry", + ) + if entry.status in BLOCKED_STATUSES: + return DeprecationCheckResult( + module_id=module_id, + status=entry.status, + is_blocked=True, + block_reason=f"Module status is '{entry.status}'", + ) + return DeprecationCheckResult( + module_id=module_id, + status=entry.status, + is_blocked=False, + block_reason="", + ) + + +def enforce_deprecation( + results: list[SimulationResult], +) -> dict[str, Any]: + total_input = len(results) + checks: list[DeprecationCheckResult] = [] + passed_results: list[SimulationResult] = [] + blocked_modules: set[str] = set() + + for result in results: + check = check_module_deprecation(result.module) + checks.append(check) + if check.is_blocked: + blocked_modules.add(result.module) + else: + passed_results.append(result) + + return { + "total_input": total_input, + "passed": len(passed_results), + "blocked": total_input - len(passed_results), + "blocked_modules": sorted(blocked_modules), + "passed_results": passed_results, + "checks": [c.to_dict() for c in checks], + "dry_lab_only": True, + } + + +def run_deprecation_check_batch( + module_ids: list[str], +) -> dict[str, Any]: + results = [check_module_deprecation(mid) for mid in module_ids] + blocked = sum(1 for r in results if r.is_blocked) + return { + "total": len(results), + "blocked": blocked, + "allowed": len(results) - blocked, + "any_blocked": blocked > 0, + "results": [r.to_dict() for r in results], + "dry_lab_only": True, + } diff --git a/tests/simulation/test_deprecation_enforcer.py b/tests/simulation/test_deprecation_enforcer.py new file mode 100644 index 00000000..d6f65da4 --- /dev/null +++ b/tests/simulation/test_deprecation_enforcer.py @@ -0,0 +1,169 @@ +"""Tests for the simulation-module deprecation enforcer.""" +from __future__ import annotations + +from openamp_foundry.simulation.deprecation_enforcer import ( + BLOCKED_STATUSES, + DeprecationCheckResult, + check_module_deprecation, + enforce_deprecation, + run_deprecation_check_batch, +) +from openamp_foundry.simulation.interfaces import SimulationResult + + +class TestDeprecationCheckResult: + def test_dry_lab_only_always_true(self): + r = DeprecationCheckResult( + module_id="test", status="active", + is_blocked=False, block_reason="", + ) + assert r.dry_lab_only is True + + def test_to_dict(self): + r = DeprecationCheckResult( + module_id="m", status="active", + is_blocked=False, block_reason="", + ) + d = r.to_dict() + assert d["module_id"] == "m" + assert d["dry_lab_only"] is True + + +class TestCheckModuleDeprecation: + def test_active_module_not_blocked(self): + r = check_module_deprecation("membrane_proxy") + assert r.is_blocked is False + assert r.status == "active" + assert r.block_reason == "" + + def test_deprecated_module_blocked(self): + r = check_module_deprecation("dummy_membrane_proxy") + assert r.is_blocked is True + assert r.status == "deprecated" + assert "deprecated" in r.block_reason + + def test_unavailable_module_blocked(self): + r = check_module_deprecation("external_adapter_placeholder") + assert r.is_blocked is True + assert r.status == "unavailable" + assert "unavailable" in r.block_reason + + def test_unknown_module_blocked_with_unknown_status(self): + r = check_module_deprecation("nonexistent_module") + assert r.is_blocked is True + assert r.status == "unknown" + assert "not found" in r.block_reason + + def test_dry_lab_only_always_true(self): + r = check_module_deprecation("membrane_proxy") + assert r.dry_lab_only is True + r2 = check_module_deprecation("dummy_membrane_proxy") + assert r2.dry_lab_only is True + r3 = check_module_deprecation("unknown") + assert r3.dry_lab_only is True + + def test_membrane_proxy_active_not_blocked(self): + r = check_module_deprecation("membrane_proxy") + assert r.is_blocked is False + assert r.status == "active" + + +class TestEnforceDeprecation: + def _make_result(self, module_id: str) -> SimulationResult: + return SimulationResult( + module=module_id, + version="0.1.0", + scope=["test"], + scores={"score": 0.5}, + uncertainty=0.1, + calibration_set=None, + validated_against=[], + ) + + def test_filters_out_deprecated_results(self): + results = [ + self._make_result("membrane_proxy"), + self._make_result("dummy_membrane_proxy"), + ] + out = enforce_deprecation(results) + assert out["blocked"] == 1 + assert len(out["passed_results"]) == 1 + assert out["passed_results"][0].module == "membrane_proxy" + + def test_passed_count_correct(self): + results = [ + self._make_result("membrane_proxy"), + self._make_result("structure_proxy"), + self._make_result("dummy_membrane_proxy"), + ] + out = enforce_deprecation(results) + assert out["total_input"] == 3 + assert out["passed"] == 2 + assert out["blocked"] == 1 + + def test_blocked_modules_sorted_and_deduplicated(self): + results = [ + self._make_result("dummy_membrane_proxy"), + self._make_result("dummy_membrane_proxy"), + self._make_result("membrane_proxy"), + ] + out = enforce_deprecation(results) + assert out["blocked_modules"] == ["dummy_membrane_proxy"] + + def test_dry_lab_only_true(self): + results = [self._make_result("membrane_proxy")] + out = enforce_deprecation(results) + assert out["dry_lab_only"] is True + + def test_all_allowed_blocked_zero(self): + results = [ + self._make_result("membrane_proxy"), + self._make_result("structure_proxy"), + ] + out = enforce_deprecation(results) + assert out["blocked"] == 0 + assert out["passed"] == 2 + + +class TestRunDeprecationCheckBatch: + def test_total_correct(self): + out = run_deprecation_check_batch( + ["membrane_proxy", "structure_proxy"] + ) + assert out["total"] == 2 + + def test_any_blocked_true_when_blocked(self): + out = run_deprecation_check_batch( + ["membrane_proxy", "dummy_membrane_proxy"] + ) + assert out["any_blocked"] is True + assert out["blocked"] == 1 + + def test_any_blocked_false_when_all_allowed(self): + out = run_deprecation_check_batch( + ["membrane_proxy", "structure_proxy"] + ) + assert out["any_blocked"] is False + assert out["blocked"] == 0 + + def test_dry_lab_only_true(self): + out = run_deprecation_check_batch(["membrane_proxy"]) + assert out["dry_lab_only"] is True + + def test_membrane_proxy_not_blocked(self): + out = run_deprecation_check_batch(["membrane_proxy"]) + assert out["any_blocked"] is False + assert out["results"][0]["is_blocked"] is False + + def test_dummy_membrane_proxy_blocked(self): + out = run_deprecation_check_batch(["dummy_membrane_proxy"]) + assert out["any_blocked"] is True + assert out["results"][0]["is_blocked"] is True + + +class TestBlockedStatuses: + def test_deprecated_in_blocked(self): + assert "deprecated" in BLOCKED_STATUSES + + def test_unavailable_in_blocked(self): + assert "unavailable" in BLOCKED_STATUSES diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index cb0f257f..9d8baa3d 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3217 tests as of Loop 95 (2026-07-09) -- Phase H H7 (simulation-result confidence interval reporter) +# Baseline: 3238 tests as of Loop 96 (2026-07-09) -- Phase H H8 (simulation-module deprecation enforcer) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3217 +BASELINE = 3238 TOLERANCE = 0.05 # allow 5% fluctuation