diff --git a/Makefile b/Makefile index 2011f939..d04c5e24 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -761,3 +761,9 @@ simulation-deprecation-check: .venv/bin/openamp-foundry simulation-deprecation-check \ --module-ids membrane_proxy,dummy_membrane_proxy,external_adapter_placeholder @echo "Deprecation check complete." + +simulation-scope-check: + .venv/bin/openamp-foundry simulation-scope-check \ + --module-id membrane_proxy \ + --requested-scopes bacterial_binding,fungal_binding + @echo "Scope check complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index eab77ece..097c3b0c 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase H H8 — simulation-module deprecation enforcer — v0.5.96) +> **Last updated:** 2026-07-09 (Phase H H9 — simulation-scope coverage checker — v0.5.97) +> **New in v0.5.97:** Simulation-scope coverage checker (H9) — `ScopeCoverageResult` dataclass (8 fields: module_id, requested_scopes, module_scopes, covered, uncovered, coverage_fraction, is_fully_covered, dry_lab_only). `check_scope_coverage()` looks up module in registry, computes covered (intersection) and uncovered (requested scopes not in module scopes), coverage_fraction = len(covered)/len(requested_scopes) if requested else 1.0. `check_result_scope()` uses conservative intersection of registry scopes and result.scope as effective module_scopes. `scope_coverage_report()` returns full dict. CLI (`openamp-foundry simulation-scope-check`) with `--module-id`, `--requested-scopes`, `--format text|json`. `make simulation-scope-check` target. 17 tests. **3255 total.** A simulation module may cover only some biological scopes. If a candidate is evaluated for a scope the module doesn't cover, that result must be flagged as out-of-scope rather than silently trusted. > **New in v0.5.96:** Simulation-module deprecation enforcer (H8) — `DeprecationCheckResult` dataclass (5 fields: module_id, status, is_blocked, block_reason, dry_lab_only). `BLOCKED_STATUSES = {"deprecated", "unavailable"}`. `check_module_deprecation()` looks up module in registry: not-found returns is_blocked=True status="unknown"; deprecated/unavailable returns is_blocked=True with reason; active/experimental returns is_blocked=False. `enforce_deprecation()` filters list[SimulationResult] to only non-blocked modules, returns total_input/passed/blocked/blocked_modules/passed_results/checks/dry_lab_only. `run_deprecation_check_batch()` bulk-checks module_ids with total/blocked/allowed/any_blocked/results/dry_lab_only. CLI (`openamp-foundry simulation-deprecation-check`) with `--module-ids`, `--format text|json`. `make simulation-deprecation-check` target. 21 tests. **3238 total.** Deprecated simulation modules must not be used in production scoring — the enforcer prevents stale or unreliable modules from tainting evidence packets. > **New in v0.5.95:** Simulation-result confidence interval reporter (H7) — `ci_reporter.py` with `ScoreCI` dataclass (9 fields: module_id, score_key, point_estimate, uncertainty, ci_lower, ci_upper, ci_width, overlaps_with, dry_lab_only). `compute_score_ci()` builds CIs from SimulationResult.scores[score_key] ± uncertainty, returns None if score_key missing. `compare_cis()` pairwise checks overlap condition (a_lo <= b_hi and b_lo <= a_hi), returns new list with overlaps_with populated (no in-place mutation). `ci_report()` produces full report with n_results, cis list, any_overlap flag, and dry_lab_only=True. CLI (`openamp-foundry simulation-ci-report`) with `--results-json`, `--score-key`, `--format text|json`. `make simulation-ci-report` target. 16 tests. **3217 total.** Raw scores without uncertainty ranges make it impossible to judge whether two candidates are distinguishable. The CI reporter makes uncertainty explicit and auditable. > **New in v0.5.94:** Simulation-ensemble agreement checker (H6) — `ensemble_checker.py` with `EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked, agreement_level, agreement_description, mean_score, score_range, scores_by_module, threshold, dry_lab_only). `AGREEMENT_LEVELS` dict (5 levels: strong, moderate, weak, conflict, insufficient). `check_ensemble_agreement()` extracts score_key from each SimulationResult's scores dict, computes score_range, classifies into strong (≥3 modules within threshold), moderate (2 modules within threshold), weak (1 module), conflict (beyond threshold), or insufficient (no results). `run_ensemble_check_batch()` aggregates multiple calls with counts and any_conflict flag. CLI (`openamp-foundry simulation-ensemble-check`) with `--sequence`, `--results-json`, `--score-key`, `--threshold`, `--format text|json`. `make simulation-ensemble-check` target. 20 tests. **3201 total.** When multiple simulation modules independently agree on a candidate, that agreement is stronger evidence than a single module alone. The ensemble checker makes this agreement explicit and auditable. @@ -87,7 +88,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.96 (H8 — simulation-module deprecation enforcer) +> **Pipeline version:** v0.5.97 (H9 — simulation-scope coverage checker) > **Branch:** main --- @@ -1157,7 +1158,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 3238 | +| Total tests | 3255 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 4a16ee56..c59e7d60 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -51,6 +51,56 @@ Honest boundaries: not invalidate results; it highlights them for human review. - All CI reports are dry-lab only and must not be presented as biological proof. +## v0.5.97 — Loop 97: Phase H H9 — Simulation-Scope Coverage Checker ✓ (2026-07-09) + +`ScopeCoverageResult` dataclass (8 fields: module_id, requested_scopes, +module_scopes, covered, uncovered, coverage_fraction, is_fully_covered, +dry_lab_only). `check_scope_coverage()` looks up module in registry, computes +covered (intersection) and uncovered (requested scopes not in module scopes), +coverage_fraction = len(covered)/len(requested_scopes) if requested else 1.0. +`check_result_scope()` uses conservative intersection of registry scopes and +result.scope as effective module_scopes. `scope_coverage_report()` returns +full dict. +CLI (`openamp-foundry simulation-scope-check`) with `--module-id`, +`--requested-scopes`, `--format text|json`. +`make simulation-scope-check` target with demo invocation. +A simulation module may cover only some biological scopes. If a candidate is +evaluated for a scope the module does not cover, that result must be flagged +as out-of-scope rather than silently trusted. + +Changes: +- `src/openamp_foundry/simulation/scope_checker.py` (H9) — Core module with + `ScopeCoverageResult` dataclass (8 fields), `check_scope_coverage()` with + registry lookup and set intersection, `check_result_scope()` with conservative + intersection of registry and result scopes, `scope_coverage_report()` dict + builder. +- `src/openamp_foundry/simulation/__init__.py` — Exports `ScopeCoverageResult`, + `check_scope_coverage`, `check_result_scope`, `scope_coverage_report`. +- `src/openamp_foundry/cli/main.py` — Registered `simulation-scope-check` + subcommand with `--module-id`, `--requested-scopes`, `--format` flags. + Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_simulation_scope_check()` CLI handler with text and JSON output. +- `Makefile` — Added `simulation-scope-check` target. Added to `.PHONY`. +- `tests/simulation/test_scope_checker.py` — 17 tests covering: fully covered, + partially covered, no scopes requested, unknown module, dry_lab_only always + True, covered/uncovered lists correct, coverage_fraction half/full/zero, + check_result_scope intersection, scope_coverage_report keys and dry_lab_only, + membrane_proxy covers bacterial, membrane_proxy does not cover fungal, empty + module scopes all uncovered. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.97 H9 changelog. Test count: 3255. +- `tests/test_test_count_regression.py` — baseline updated to 3255. + +Honest boundaries: +- Scope coverage is based on the module registry's declared scopes, which may + be incomplete or outdated relative to a module's actual capabilities. +- `check_result_scope` is conservative: it only trusts scopes that both the + registry and the result agree on. This may undercount coverage if result + metadata is sparse. +- A fully covered scope does not mean the simulation is accurate or biologically + meaningful — only that the module claims to cover the requested scope. +- All outputs are dry-lab only and must not be presented as biological proof. + ## v0.5.96 — Loop 96: Phase H H8 — Simulation-Module Deprecation Enforcer ✓ (2026-07-09) `DeprecationCheckResult` dataclass (5 fields: module_id, status, is_blocked, diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 542b27a0..68ab7111 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1741,3 +1741,36 @@ def _run_simulation_deprecation_check(args: argparse.Namespace) -> int: print("not biological proof.") return 3 if result["any_blocked"] else 0 + + +def _run_simulation_scope_check(args: argparse.Namespace) -> int: + """Check whether a simulation module covers all requested scopes.""" + from openamp_foundry.simulation.scope_checker import ( + check_scope_coverage, + ) + + module_id = args.module_id + requested_scopes = [s.strip() for s in args.requested_scopes.split(",") if s.strip()] + output_format = getattr(args, "format", "text") + + if not requested_scopes: + print({"status": "error", "error": "No scopes provided."}) + return 2 + + result = check_scope_coverage(module_id, requested_scopes) + + if output_format == "json": + print(json.dumps(result.to_dict(), indent=2)) + else: + print(f"Simulation Scope Coverage Check: {module_id}") + print(f" Requested scopes: {', '.join(result.requested_scopes)}") + print(f" Module scopes: {', '.join(result.module_scopes) if result.module_scopes else '(none)'}") + print(f" Covered: {', '.join(result.covered) if result.covered else '(none)'}") + print(f" Uncovered: {', '.join(result.uncovered) if result.uncovered else '(none)'}") + print(f" Coverage: {result.coverage_fraction:.1%}") + print(f" Fully covered: {result.is_fully_covered}") + print() + print("Dry-lab only. Scope coverage checks are computational safeguards,") + print("not biological proof.") + + return 0 if result.is_fully_covered else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 76e8685c..525fa4cd 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1538,6 +1538,29 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Simulation-scope coverage check (H9) ───────────────────────── + ssc = sub.add_parser( + "simulation-scope-check", + help=( + "Check whether a simulation module covers all requested biological scopes. " + "Flags out-of-scope results so they are not silently trusted. " + "Dry-lab only." + ), + ) + ssc.add_argument( + "--module-id", type=str, required=True, + help="Module ID to check (e.g. membrane_proxy).", + ) + ssc.add_argument( + "--requested-scopes", type=str, required=True, + help="Comma-separated scopes (e.g. bacterial_binding,fungal_binding).", + ) + ssc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Simulation-result confidence interval report (H7) ──────────── sicr = sub.add_parser( "simulation-ci-report", @@ -1740,6 +1763,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "simulation-deprecation-check": return _run_simulation_deprecation_check(args) + if args.command == "simulation-scope-check": + return _run_simulation_scope_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index 0b48c3e2..ef2b4c47 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -1,5 +1,11 @@ """Virtual assay simulation layer for OpenAMP Foundry.""" +from .scope_checker import ( + ScopeCoverageResult, + check_scope_coverage, + check_result_scope, + scope_coverage_report, +) from .deprecation_enforcer import ( BLOCKED_STATUSES, DeprecationCheckResult, @@ -63,6 +69,10 @@ from .structure import StructureProxy __all__ = [ + "ScopeCoverageResult", + "check_scope_coverage", + "check_result_scope", + "scope_coverage_report", "BLOCKED_STATUSES", "DeprecationCheckResult", "check_module_deprecation", diff --git a/src/openamp_foundry/simulation/scope_checker.py b/src/openamp_foundry/simulation/scope_checker.py new file mode 100644 index 00000000..910cc6c0 --- /dev/null +++ b/src/openamp_foundry/simulation/scope_checker.py @@ -0,0 +1,79 @@ +"""Simulation-scope coverage checker — flags out-of-scope results as uncovered. + +Dry-lab only. Scope checks are computational safeguards, not biological proof. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, asdict +from typing import Any + +from openamp_foundry.simulation.interfaces import SimulationResult +from openamp_foundry.simulation.module_registry import get_module_entry + + +@dataclass +class ScopeCoverageResult: + module_id: str + requested_scopes: list[str] + module_scopes: list[str] + covered: list[str] + uncovered: list[str] + coverage_fraction: float + is_fully_covered: bool + dry_lab_only: bool = True + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def check_scope_coverage( + module_id: str, + requested_scopes: list[str], +) -> ScopeCoverageResult: + entry = get_module_entry(module_id) + module_scopes = list(entry.scope) if entry is not None else [] + covered = [s for s in requested_scopes if s in module_scopes] + uncovered = [s for s in requested_scopes if s not in module_scopes] + coverage_fraction = len(covered) / len(requested_scopes) if requested_scopes else 1.0 + is_fully_covered = len(uncovered) == 0 + return ScopeCoverageResult( + module_id=module_id, + requested_scopes=list(requested_scopes), + module_scopes=module_scopes, + covered=covered, + uncovered=uncovered, + coverage_fraction=coverage_fraction, + is_fully_covered=is_fully_covered, + ) + + +def check_result_scope( + result: SimulationResult, + requested_scopes: list[str], +) -> ScopeCoverageResult: + entry = get_module_entry(result.module) + registry_scopes = set(entry.scope) if entry is not None else set() + result_scopes = set(result.scope) + module_scopes = sorted(registry_scopes & result_scopes) + covered = [s for s in requested_scopes if s in module_scopes] + uncovered = [s for s in requested_scopes if s not in module_scopes] + coverage_fraction = len(covered) / len(requested_scopes) if requested_scopes else 1.0 + is_fully_covered = len(uncovered) == 0 + return ScopeCoverageResult( + module_id=result.module, + requested_scopes=list(requested_scopes), + module_scopes=module_scopes, + covered=covered, + uncovered=uncovered, + coverage_fraction=coverage_fraction, + is_fully_covered=is_fully_covered, + ) + + +def scope_coverage_report( + module_id: str, + requested_scopes: list[str], +) -> dict[str, Any]: + result = check_scope_coverage(module_id, requested_scopes) + return result.to_dict() diff --git a/tests/simulation/test_scope_checker.py b/tests/simulation/test_scope_checker.py new file mode 100644 index 00000000..8ab66d72 --- /dev/null +++ b/tests/simulation/test_scope_checker.py @@ -0,0 +1,111 @@ +"""Tests for simulation-scope coverage checker.""" +import json +import pytest + +from openamp_foundry.simulation.interfaces import SimulationResult +from openamp_foundry.simulation.scope_checker import ( + ScopeCoverageResult, + check_scope_coverage, + check_result_scope, + scope_coverage_report, +) + + +class TestScopeCoverageResult: + def test_fully_covered(self): + result = check_scope_coverage("membrane_proxy", ["bacterial_membrane_binding"]) + assert result.is_fully_covered is True + assert result.covered == ["bacterial_membrane_binding"] + assert result.uncovered == [] + + def test_partially_covered(self): + result = check_scope_coverage("membrane_proxy", ["bacterial_membrane_binding", "fungal_binding"]) + assert result.is_fully_covered is False + assert result.uncovered == ["fungal_binding"] + assert result.covered == ["bacterial_membrane_binding"] + + def test_no_scopes_requested(self): + result = check_scope_coverage("membrane_proxy", []) + assert result.coverage_fraction == 1.0 + assert result.is_fully_covered is True + + def test_unknown_module_id(self): + result = check_scope_coverage("nonexistent_module", ["bacterial_membrane_binding"]) + assert result.module_scopes == [] + assert result.uncovered == ["bacterial_membrane_binding"] + assert result.is_fully_covered is False + + @pytest.mark.parametrize("func", [ + check_scope_coverage, + lambda m, s: check_result_scope( + SimulationResult( + module=m, version="0.1.0", scope=["bacterial_membrane_binding"], + scores={"x": 0.5}, uncertainty=0.1, calibration_set=None, + validated_against=[], notes=[], + ), s, + ), + ]) + def test_dry_lab_only_always_true(self, func): + if func == check_scope_coverage: + result = func("membrane_proxy", ["bacterial_membrane_binding"]) + else: + result = func("membrane_proxy", ["bacterial_membrane_binding"]) + assert result.dry_lab_only is True + + def test_covered_list_correct(self): + result = check_scope_coverage("membrane_proxy", ["bacterial_membrane_binding", "fungal_binding"]) + assert result.covered == ["bacterial_membrane_binding"] + + def test_uncovered_list_correct(self): + result = check_scope_coverage("membrane_proxy", ["bacterial_membrane_binding", "fungal_binding"]) + assert result.uncovered == ["fungal_binding"] + + def test_coverage_fraction_half(self): + result = check_scope_coverage("membrane_proxy", ["bacterial_membrane_binding", "fungal_binding"]) + assert result.coverage_fraction == 0.5 + + def test_coverage_fraction_full(self): + result = check_scope_coverage("membrane_proxy", ["bacterial_membrane_binding"]) + assert result.coverage_fraction == 1.0 + + def test_coverage_fraction_zero(self): + result = check_scope_coverage("structure_proxy", ["bacterial_membrane_binding"]) + assert result.coverage_fraction == 0.0 + + def test_check_result_scope_intersection(self): + result = SimulationResult( + module="membrane_proxy", version="0.1.0", + scope=["bacterial_membrane_binding", "extra_scope"], + scores={"x": 0.5}, uncertainty=0.1, calibration_set=None, + validated_against=[], notes=[], + ) + coverage = check_result_scope(result, ["bacterial_membrane_binding", "extra_scope"]) + assert coverage.module_scopes == ["bacterial_membrane_binding"] + assert coverage.covered == ["bacterial_membrane_binding"] + assert coverage.uncovered == ["extra_scope"] + + def test_scope_coverage_report_keys(self): + report = scope_coverage_report("membrane_proxy", ["bacterial_membrane_binding"]) + expected_keys = { + "module_id", "requested_scopes", "module_scopes", + "covered", "uncovered", "coverage_fraction", + "is_fully_covered", "dry_lab_only", + } + assert set(report.keys()) == expected_keys + + def test_scope_coverage_report_dry_lab_only(self): + report = scope_coverage_report("membrane_proxy", ["bacterial_membrane_binding"]) + assert report["dry_lab_only"] is True + + def test_membrane_proxy_covers_bacterial(self): + result = check_scope_coverage("membrane_proxy", ["bacterial_membrane_binding"]) + assert "bacterial_membrane_binding" in result.covered + + def test_membrane_proxy_does_not_cover_fungal(self): + result = check_scope_coverage("membrane_proxy", ["fungal_binding"]) + assert "fungal_binding" in result.uncovered + + def test_empty_module_scopes_all_uncovered(self): + result = check_scope_coverage("external_adapter_placeholder", ["fungal_binding"]) + assert result.uncovered == ["fungal_binding"] + assert result.coverage_fraction == 0.0 diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 9d8baa3d..6660eff7 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3238 tests as of Loop 96 (2026-07-09) -- Phase H H8 (simulation-module deprecation enforcer) +# Baseline: 3255 tests as of Loop 97 (2026-07-09) -- Phase H H9 (simulation-scope coverage checker) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3238 +BASELINE = 3255 TOLERANCE = 0.05 # allow 5% fluctuation