From a1fa7bef2c94559620ad18b17bed34997b7364cc Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 13:28:40 +0700 Subject: [PATCH] feat: Phase H H4 fail-closed adapter integration tests -- avoids hidden external failures --- Makefile | 10 +- docs/evidence/METRICS_CURRENT.md | 7 +- docs/research/ROADMAP.md | 51 ++++++ src/openamp_foundry/cli/commands/reports.py | 62 +++++++ src/openamp_foundry/cli/main.py | 44 ++++- src/openamp_foundry/simulation/__init__.py | 10 ++ .../simulation/adapter_gate.py | 144 ++++++++++++++++ tests/simulation/test_adapter_gate.py | 162 ++++++++++++++++++ tests/test_test_count_regression.py | 4 +- 9 files changed, 487 insertions(+), 7 deletions(-) create mode 100644 src/openamp_foundry/simulation/adapter_gate.py create mode 100644 tests/simulation/test_adapter_gate.py diff --git a/Makefile b/Makefile index 0efb3b9d..f7bfe770 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -727,3 +727,11 @@ simulation-baseline-check: --claimed-level 2 \ --baseline-beaten false @echo "Baseline check complete." + +adapter-gate-check: + .venv/bin/openamp-foundry adapter-gate-check \ + --module-id membrane_proxy \ + --timeout false \ + --schema-errors "[]" \ + --baseline-beaten false + @echo "Adapter gate check complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 4f61d853..7efd8736 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase H H3 — per-module cheapest-baseline declaration — v0.5.91) +> **Last updated:** 2026-07-09 (Phase H H4 — fail-closed adapter integration tests — v0.5.92) +> **New in v0.5.92:** Fail-closed adapter integration tests (H4) — `FAIL_CLOSED_REASONS` dict keys (6 entries: timeout, connection_refused, invalid_response, schema_violation, module_unavailable, baseline_not_beaten). `AdapterGateResult` dataclass with module_id, passed, failure_reason, failure_detail, dry_lab_only. `evaluate_adapter_gate()` fail-closed: returns passed=False on ANY failure signal with priority ordering (timeout > connection_refused > invalid_response > schema_violation > module_unavailable > baseline_not_beaten). `run_adapter_gate_batch()` aggregates multiple adapter calls with total/passed/failed/any_failed/results/dry_lab_only. CLI (`openamp-foundry adapter-gate-check`) with `--module-id`, `--timeout`, `--connection-refused`, `--schema-errors`, `--module-unavailable`, `--baseline-beaten`, `--format text|json`. `make adapter-gate-check` target. 21 tests. **3162 total.** Avoids hidden external failures — when the adapter to an external simulation service is down or misbehaves, the pipeline must fail loudly rather than silently passing garbage through. > **New in v0.5.91:** Per-module cheapest-baseline declaration (H3) — `BaselineDeclaration` dataclass with module_id, module_name, baseline_description, baseline_type, evidence_level_ceiling, and notes. `BASELINE_DECLARATIONS` list (4 entries: membrane_proxy, structure_proxy, dummy_membrane_proxy, external_adapter_placeholder). `get_baseline_declaration()` and `list_baseline_declarations()` for lookup. `check_baseline_requirement()` caps effective_evidence_level to ceiling when baseline not beaten; returns dict with module_id, baseline_beaten, claimed/effective evidence levels, capped flag, message, and dry_lab_only. `validate_baseline_declarations()` checks module_id non-empty, baseline_description non-empty, baseline_type in {heuristic, random, constant, length}, evidence_level_ceiling 1-6, no duplicate module_ids. CLI (`openamp-foundry simulation-baseline-check`) with `--module-id`, `--claimed-level`, `--baseline-beaten`, `--format text|json`. `make simulation-baseline-check` target. 16 tests. **3141 total.** Forces honest enemy comparison — every simulation module must declare the simplest baseline it must beat. > **New in v0.5.90:** Simulation-result schema and validator (H2) — `schemas/simulation_result.schema.json` (Draft 2020-12) validates SimulationResult outputs with uncertainty 0.0–1.0 range, required fields (module, version, scope, scores, uncertainty, calibration_set, validated_against, notes), and optional `dry_lab_context` const "dry-lab-only". `validate_simulation_result()` checks module, version, scope, scores, uncertainty, validated_against, notes. Strict mode adds: module must not be "dummy" or contain "stub", uncertainty < 1.0, validated_against non-empty. `validate_simulation_result_batch()` aggregates results with counts and `any_invalid` flag. CLI (`openamp-foundry validate-simulation-result`) with `--results-json`, `--strict`, `--out-json`. `make validate-simulation-result-schema` target. 19 tests. **3125 total.** Dry-lab only. > **New in v0.5.89:** Simulation module registry (H1) — `SimulationModuleEntry` dataclass tracks module_id, name, description, status, evidence_level, baseline_comparison, scope, maintainer, and notes. `SIMULATION_MODULE_REGISTRY` holds 4 entries (membrane_proxy, structure_proxy, dummy_membrane_proxy, external_adapter_placeholder). Lookup functions: `get_module_entry()`, `list_module_entries()` with status/min_evidence filtering, `get_active_modules()`, `registry_summary()` with total/by_status/by_evidence_level/active_module_ids keys. `validate_registry()` checks module_id, name, baseline_comparison, evidence_level 1-6, valid status, duplicate detection. CLI (`openamp-foundry simulation-registry`) supports `--list`, `--show`, `--status`, `--min-evidence`, `--format text|json`. Schema (`schemas/simulation_module_registry.schema.json`). `make simulation-registry` target. 28 tests. 3106 total. Starts Phase H (virtual assay discipline). @@ -82,7 +83,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.91 (H3 — per-module cheapest-baseline declaration) +> **Pipeline version:** v0.5.92 (H4 — fail-closed adapter integration tests) > **Branch:** main --- @@ -1152,7 +1153,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 3125 | +| Total tests | 3161 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 428def1a..30b840fc 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,56 @@ # Roadmap +## v0.5.92 — Loop 92: Phase H H4 — Fail-Closed Adapter Integration Tests ✓ (2026-07-09) + +`FAIL_CLOSED_REASONS` dict (6 keys) enumerates known adapter failure reasons. +`AdapterGateResult` dataclass with module_id, passed, failure_reason, failure_detail, +dry_lab_only. `evaluate_adapter_gate()` fail-closed: returns passed=False on ANY +failure signal with strict priority ordering (timeout > connection_refused > +invalid_response > schema_violation > module_unavailable > baseline_not_beaten). +`run_adapter_gate_batch()` aggregates multiple adapter calls with total/passed/failed/ +any_failed/results/dry_lab_only. Avoids hidden external failures — when the adapter +to an external simulation service is down or misbehaves, the pipeline must fail +loudly rather than silently passing garbage through. + +Changes: +- `src/openamp_foundry/simulation/adapter_gate.py` (H4) — Core module with + `FAIL_CLOSED_REASONS` dict (6 entries), `AdapterGateResult` dataclass (5 fields: + module_id, passed, failure_reason, failure_detail, dry_lab_only), + `evaluate_adapter_gate()` with 7-path priority logic, + `run_adapter_gate_batch()` aggregating results with counts and any_failed flag. +- `src/openamp_foundry/simulation/__init__.py` — Exports `AdapterGateResult`, + `FAIL_CLOSED_REASONS`, `evaluate_adapter_gate`, `run_adapter_gate_batch`. +- `src/openamp_foundry/cli/main.py` — Registered `adapter-gate-check` subcommand + with `--module-id`, `--timeout`, `--connection-refused`, `--schema-errors`, + `--module-unavailable`, `--baseline-beaten`, `--format` flags. Added import + and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` — Added `_run_adapter_gate_check()` + CLI handler with text and JSON output, JSON parsing for schema errors, + exit code 3 on failure. +- `Makefile` — Added `adapter-gate-check` target with default `membrane_proxy` + check. Added to `.PHONY`. +- `tests/simulation/test_adapter_gate.py` — 20 tests covering: all 6 failure + reasons, all-clear pass, dry_lab_only always True, priority ordering (timeout + beats connection_refused, connection_refused beats result=None, schema_errors + beat module_unavailable), baseline_beaten=True passes, baseline_beaten=None + does not trigger, batch counts, batch any_failed, batch dry_lab_only. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.92 H4 changelog. Test count: 3161. +- `tests/test_test_count_regression.py` — baseline updated to 3161. + +Honest boundaries: +- The adapter gate checks for known failure signals only. An adapter that + returns plausible-looking but biologically meaningless results will pass. +- Failure detection depends on the caller correctly setting the failure + flags. An adapter that silently hangs (neither timeout nor connection + refused) may not be caught. +- Schema validation errors detect structural contract violations, not + biological correctness. A schema-valid response can still be biologically + meaningless. +- The `baseline_not_beaten` check requires the caller to run the baseline + comparison externally; the gate does not compute it. +- All adapter gate checks are dry-lab only and must not be presented as + biological proof. + ## v0.5.91 — Loop 91: Phase H H3 — Per-Module Cheapest-Baseline Declaration ✓ (2026-07-09) `BaselineDeclaration` dataclass with module_id, module_name, baseline_description, diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 25ad58b5..aad83db8 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1433,3 +1433,65 @@ def _run_simulation_baseline_check(args: argparse.Namespace) -> int: print("not biological proof.") return 3 if result["capped"] else 0 + + +def _run_adapter_gate_check(args: argparse.Namespace) -> int: + """Fail-closed adapter gate check for simulation adapters.""" + from openamp_foundry.simulation.adapter_gate import ( + evaluate_adapter_gate, + ) + + module_id = args.module_id + timeout_occurred = args.timeout.lower() == "true" + connection_refused = args.connection_refused.lower() == "true" + module_unavailable = args.module_unavailable.lower() == "true" + output_format = getattr(args, "format", "text") + + schema_errors_raw = args.schema_errors + try: + schema_errors = json.loads(schema_errors_raw) if schema_errors_raw else [] + except (json.JSONDecodeError, TypeError): + print(json.dumps({ + "status": "error", + "error": f"Invalid JSON for --schema-errors: {schema_errors_raw!r}", + })) + return 2 + + if not isinstance(schema_errors, list): + print(json.dumps({ + "status": "error", + "error": "--schema-errors must be a JSON array of strings", + })) + return 2 + + baseline_beaten_raw = args.baseline_beaten + baseline_beaten: bool | None = None + if baseline_beaten_raw is not None: + baseline_beaten = baseline_beaten_raw.lower() == "true" + + gate_result = evaluate_adapter_gate( + module_id=module_id, + result=None, + timeout_occurred=timeout_occurred, + connection_refused=connection_refused, + schema_errors=schema_errors, + module_unavailable=module_unavailable, + baseline_beaten=baseline_beaten, + ) + + if output_format == "json": + print(json.dumps(gate_result.to_dict(), indent=2)) + else: + status = "PASS" if gate_result.passed else "FAIL" + reason_info = "" + if gate_result.failure_reason: + reason_info = f" Reason: {gate_result.failure_reason} — {gate_result.failure_detail}" + print(f"Adapter Gate Check: {module_id}") + print(f" Status: {status}") + if reason_info: + print(reason_info) + print() + print("Dry-lab only. Adapter gate checks are computational safeguards,") + print("not biological proof.") + + return 0 if gate_result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 53edd404..64fc0c87 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1430,6 +1430,45 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Adapter gate check (H4) ────────────────────────────────────── + agc = sub.add_parser( + "adapter-gate-check", + help=( + "Check whether a simulation adapter passes the fail-closed gate. " + "Dry-lab only." + ), + ) + agc.add_argument("--module-id", type=str, required=True, help="Module ID to check.") + agc.add_argument( + "--timeout", type=str, default="false", + choices=["true", "false"], + help="Whether a timeout occurred (default: false).", + ) + agc.add_argument( + "--connection-refused", type=str, default="false", + choices=["true", "false"], + help="Whether connection was refused (default: false).", + ) + agc.add_argument( + "--schema-errors", type=str, default="[]", + help="JSON array of schema error strings (default: []).", + ) + agc.add_argument( + "--module-unavailable", type=str, default="false", + choices=["true", "false"], + help="Whether the module is unavailable (default: false).", + ) + agc.add_argument( + "--baseline-beaten", type=str, default=None, + choices=["true", "false"], + help="Whether the baseline was beaten (optional).", + ) + agc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + return parser @@ -1594,6 +1633,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "simulation-baseline-check": return _run_simulation_baseline_check(args) + if args.command == "adapter-gate-check": + return _run_adapter_gate_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index a1ec5245..57da86b5 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -1,5 +1,11 @@ """Virtual assay simulation layer for OpenAMP Foundry.""" +from .adapter_gate import ( + AdapterGateResult, + FAIL_CLOSED_REASONS, + evaluate_adapter_gate, + run_adapter_gate_batch, +) from .baseline_registry import ( BASELINE_DECLARATIONS, BaselineDeclaration, @@ -32,6 +38,10 @@ from .structure import StructureProxy __all__ = [ + "AdapterGateResult", + "FAIL_CLOSED_REASONS", + "evaluate_adapter_gate", + "run_adapter_gate_batch", "SimulationResult", "VirtualAssayProxy", "ExternalSimulationAdapter", diff --git a/src/openamp_foundry/simulation/adapter_gate.py b/src/openamp_foundry/simulation/adapter_gate.py new file mode 100644 index 00000000..ea83c91b --- /dev/null +++ b/src/openamp_foundry/simulation/adapter_gate.py @@ -0,0 +1,144 @@ +"""Fail-closed adapter gate for virtual assay simulation adapters. + +When an adapter to an external simulation service is down or misbehaves, +the pipeline must fail loudly rather than silently passing garbage through. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, asdict +from typing import Any + +from .interfaces import SimulationResult + +FAIL_CLOSED_REASONS: dict[str, str] = { + "timeout": "Adapter call timed out", + "connection_refused": "Adapter endpoint refused connection", + "invalid_response": "Adapter returned an unparseable response", + "schema_violation": "Adapter response failed schema validation", + "module_unavailable": "Simulation module is not available", + "baseline_not_beaten": "Module did not beat its declared cheapest baseline", +} + + +@dataclass +class AdapterGateResult: + module_id: str + passed: bool + failure_reason: str | None + failure_detail: str + dry_lab_only: bool = True + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def evaluate_adapter_gate( + module_id: str, + result: SimulationResult | None, + *, + timeout_occurred: bool = False, + connection_refused: bool = False, + schema_errors: list[str] | None = None, + module_unavailable: bool = False, + baseline_beaten: bool | None = None, +) -> AdapterGateResult: + """Fail-closed: return passed=False on ANY failure signal. + + Priority order (first match wins): + 1. timeout_occurred → failure_reason="timeout" + 2. connection_refused → failure_reason="connection_refused" + 3. result is None → failure_reason="invalid_response" + 4. schema_errors (non-empty list) → failure_reason="schema_violation" + 5. module_unavailable → failure_reason="module_unavailable" + 6. baseline_beaten is False → failure_reason="baseline_not_beaten" + 7. Otherwise → passed=True + """ + if timeout_occurred: + return AdapterGateResult( + module_id=module_id, + passed=False, + failure_reason="timeout", + failure_detail=FAIL_CLOSED_REASONS["timeout"], + ) + + if connection_refused: + return AdapterGateResult( + module_id=module_id, + passed=False, + failure_reason="connection_refused", + failure_detail=FAIL_CLOSED_REASONS["connection_refused"], + ) + + if result is None: + return AdapterGateResult( + module_id=module_id, + passed=False, + failure_reason="invalid_response", + failure_detail=FAIL_CLOSED_REASONS["invalid_response"], + ) + + if schema_errors: + detail = f"{FAIL_CLOSED_REASONS['schema_violation']}: {'; '.join(schema_errors)}" + return AdapterGateResult( + module_id=module_id, + passed=False, + failure_reason="schema_violation", + failure_detail=detail, + ) + + if module_unavailable: + return AdapterGateResult( + module_id=module_id, + passed=False, + failure_reason="module_unavailable", + failure_detail=FAIL_CLOSED_REASONS["module_unavailable"], + ) + + if baseline_beaten is False: + return AdapterGateResult( + module_id=module_id, + passed=False, + failure_reason="baseline_not_beaten", + failure_detail=FAIL_CLOSED_REASONS["baseline_not_beaten"], + ) + + return AdapterGateResult( + module_id=module_id, + passed=True, + failure_reason=None, + failure_detail="All adapter gate checks passed.", + ) + + +def run_adapter_gate_batch(calls: list[dict]) -> dict: + """Run evaluate_adapter_gate for multiple calls and aggregate results. + + Each dict in calls must have a "module_id" key plus any kwargs for + evaluate_adapter_gate (result, timeout_occurred, connection_refused, + schema_errors, module_unavailable, baseline_beaten). + """ + results: list[dict] = [] + for call in calls: + module_id = call.pop("module_id") + result = call.pop("result", None) + # Pop known kwargs; remaining are passed as-is + gate_result = evaluate_adapter_gate( + module_id=module_id, + result=result, + **call, + ) + results.append(gate_result.to_dict()) + + total = len(results) + passed = sum(1 for r in results if r["passed"]) + failed = total - passed + + return { + "total": total, + "passed": passed, + "failed": failed, + "any_failed": failed > 0, + "results": results, + "dry_lab_only": True, + } diff --git a/tests/simulation/test_adapter_gate.py b/tests/simulation/test_adapter_gate.py new file mode 100644 index 00000000..4ab417e1 --- /dev/null +++ b/tests/simulation/test_adapter_gate.py @@ -0,0 +1,162 @@ +"""Tests for fail-closed adapter gate.""" +from __future__ import annotations + +import pytest + +from openamp_foundry.simulation.adapter_gate import ( + FAIL_CLOSED_REASONS, + AdapterGateResult, + evaluate_adapter_gate, + run_adapter_gate_batch, +) +from openamp_foundry.simulation.interfaces import SimulationResult + + +def _make_result(**overrides) -> SimulationResult: + base = SimulationResult( + module="membrane_proxy", + version="0.1.0", + scope=["bacterial_binding"], + scores={"binding_energy": 0.75}, + uncertainty=0.15, + calibration_set="demo_amps_2026", + validated_against=["APD3"], + notes=["Test result."], + ) + for k, v in overrides.items(): + setattr(base, k, v) + return base + + +class TestEvaluateAdapterGate: + def test_timeout_fails_closed(self): + r = evaluate_adapter_gate("m1", _make_result(), timeout_occurred=True) + assert r.passed is False + assert r.failure_reason == "timeout" + assert r.dry_lab_only is True + + def test_connection_refused_fails_closed(self): + r = evaluate_adapter_gate("m1", _make_result(), connection_refused=True) + assert r.passed is False + assert r.failure_reason == "connection_refused" + + def test_result_none_fails_invalid_response(self): + r = evaluate_adapter_gate("m1", None) + assert r.passed is False + assert r.failure_reason == "invalid_response" + + def test_schema_errors_fails_schema_violation(self): + r = evaluate_adapter_gate("m1", _make_result(), schema_errors=["field missing"]) + assert r.passed is False + assert r.failure_reason == "schema_violation" + + def test_module_unavailable_fails(self): + r = evaluate_adapter_gate("m1", _make_result(), module_unavailable=True) + assert r.passed is False + assert r.failure_reason == "module_unavailable" + + def test_baseline_not_beaten_fails(self): + r = evaluate_adapter_gate("m1", _make_result(), baseline_beaten=False) + assert r.passed is False + assert r.failure_reason == "baseline_not_beaten" + + def test_all_clear_passes(self): + r = evaluate_adapter_gate("m1", _make_result(), baseline_beaten=True) + assert r.passed is True + assert r.failure_reason is None + + def test_dry_lab_only_always_true(self): + for passed in [True, False]: + r = evaluate_adapter_gate( + "m1", + _make_result() if passed else None, + ) + assert r.dry_lab_only is True + + def test_priority_timeout_beats_connection_refused(self): + r = evaluate_adapter_gate( + "m1", _make_result(), + timeout_occurred=True, + connection_refused=True, + ) + assert r.failure_reason == "timeout" + + def test_priority_connection_refused_beats_result_none(self): + r = evaluate_adapter_gate( + "m1", None, + connection_refused=True, + ) + assert r.failure_reason == "connection_refused" + + def test_priority_schema_errors_beat_module_unavailable(self): + r = evaluate_adapter_gate( + "m1", _make_result(), + schema_errors=["bad field"], + module_unavailable=True, + ) + assert r.failure_reason == "schema_violation" + + def test_baseline_beaten_true_does_not_fail(self): + r = evaluate_adapter_gate("m1", _make_result(), baseline_beaten=True) + assert r.passed is True + + def test_baseline_beaten_none_does_not_trigger_baseline_check(self): + r = evaluate_adapter_gate("m1", _make_result(), baseline_beaten=None) + assert r.passed is True + assert r.failure_reason is None + + def test_empty_schema_errors_does_not_fail(self): + r = evaluate_adapter_gate("m1", _make_result(), schema_errors=[]) + assert r.passed is True + + +class TestRunAdapterGateBatch: + def test_batch_with_mix_correct_counts(self): + calls = [ + {"module_id": "m1", "result": _make_result(), "baseline_beaten": True}, + {"module_id": "m2", "result": None}, + {"module_id": "m3", "result": _make_result(), "timeout_occurred": True}, + ] + out = run_adapter_gate_batch(calls) + assert out["total"] == 3 + assert out["passed"] == 1 + assert out["failed"] == 2 + + def test_batch_any_failed_true_when_any_fail(self): + calls = [ + {"module_id": "m1", "result": None}, + ] + out = run_adapter_gate_batch(calls) + assert out["any_failed"] is True + + def test_batch_any_failed_false_when_all_pass(self): + calls = [ + {"module_id": "m1", "result": _make_result(), "baseline_beaten": True}, + ] + out = run_adapter_gate_batch(calls) + assert out["any_failed"] is False + + def test_batch_dry_lab_only_true(self): + out = run_adapter_gate_batch([]) + assert out["dry_lab_only"] is True + + def test_batch_results_list_length(self): + calls = [ + {"module_id": "m1", "result": _make_result()}, + {"module_id": "m2", "result": _make_result()}, + ] + out = run_adapter_gate_batch(calls) + assert len(out["results"]) == 2 + + +class TestFailClosedReasons: + def test_fail_closed_reasons_has_required_keys(self): + required = { + "timeout", "connection_refused", "invalid_response", + "schema_violation", "module_unavailable", "baseline_not_beaten", + } + assert required.issubset(set(FAIL_CLOSED_REASONS.keys())) + + def test_all_reasons_have_non_empty_descriptions(self): + for key, desc in FAIL_CLOSED_REASONS.items(): + assert desc, f"Empty description for reason: {key}" diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index fbda6fa9..b688b5eb 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3141 tests as of Loop 91 (2026-07-09) -- Phase H H3 (per-module cheapest-baseline declaration) +# Baseline: 3161 tests as of Loop 92 (2026-07-09) -- Phase H H4 (fail-closed adapter integration tests) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3141 +BASELINE = 3161 TOLERANCE = 0.05 # allow 5% fluctuation