From 56b2801e654a9bb8a0081c8a1c9ebc28bd6d4d99 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 14:05:03 +0700 Subject: [PATCH] feat: Phase H H10 simulation-evidence packet assembler -- Phase H capstone, auditable evidence packets --- Makefile | 11 +- docs/evidence/METRICS_CURRENT.md | 5 +- docs/research/ROADMAP.md | 62 ++++ src/openamp_foundry/cli/commands/reports.py | 81 ++++++ src/openamp_foundry/cli/main.py | 42 ++- src/openamp_foundry/simulation/__init__.py | 8 + .../simulation/evidence_packet.py | 128 +++++++++ tests/simulation/test_evidence_packet.py | 272 ++++++++++++++++++ tests/test_test_count_regression.py | 4 +- 9 files changed, 607 insertions(+), 6 deletions(-) create mode 100644 src/openamp_foundry/simulation/evidence_packet.py create mode 100644 tests/simulation/test_evidence_packet.py diff --git a/Makefile b/Makefile index d04c5e24..8ee7c966 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -767,3 +767,12 @@ simulation-scope-check: --module-id membrane_proxy \ --requested-scopes bacterial_binding,fungal_binding @echo "Scope check complete." + +simulation-evidence-packet: + .venv/bin/openamp-foundry simulation-evidence-packet \ + --module-id membrane_proxy \ + --result-json '{"module":"membrane_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.75},"uncertainty":0.1,"calibration_set":null,"validated_against":["APD3"],"notes":[]}' \ + --requested-scopes bacterial_binding \ + --claimed-level 2 \ + --baseline-beaten false + @echo "Evidence packet complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 097c3b0c..493117f8 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase H H9 — simulation-scope coverage checker — v0.5.97) +> **Last updated:** 2026-07-09 (Phase H H10 — simulation-evidence packet assembler — v0.5.98) +> **New in v0.5.98:** Simulation-evidence packet assembler (H10 — Phase H capstone) — `SimulationEvidencePacket` dataclass (12 fields: module_id, result, requested_scopes, claimed_evidence_level, baseline_beaten, deprecation_check, scope_check, baseline_check, adapter_gate, effective_evidence_level, all_checks_passed, failure_reasons, dry_lab_only). `assemble_evidence_packet()` runs all Phase H sub-checks (deprecation, scope, baseline, adapter gate) and assembles into a single auditable packet with all_checks_passed (True only if not deprecation_check["is_blocked"], scope_check["is_fully_covered"], not baseline_check["capped"], and adapter_gate["passed"]), failure_reasons (human-readable list of what failed), and effective_evidence_level (from baseline_check). `evidence_packet_summary()` returns compact dict with module_id, claimed_evidence_level, effective_evidence_level, all_checks_passed, failure_reasons, dry_lab_only. CLI (`openamp-foundry simulation-evidence-packet`) with `--module-id`, `--result-json`, `--requested-scopes`, `--claimed-level`, `--baseline-beaten`, `--format text|json`. `make simulation-evidence-packet` target. 16+ tests. **3272 total.** This is the capstone of Phase H — it assembles all the individual simulation discipline checks into a single auditable evidence packet showing exactly why a simulation result is trustworthy (or not) enough to support a given evidence level claim. > **New in v0.5.97:** Simulation-scope coverage checker (H9) — `ScopeCoverageResult` dataclass (8 fields: module_id, requested_scopes, module_scopes, covered, uncovered, coverage_fraction, is_fully_covered, dry_lab_only). `check_scope_coverage()` looks up module in registry, computes covered (intersection) and uncovered (requested scopes not in module scopes), coverage_fraction = len(covered)/len(requested_scopes) if requested else 1.0. `check_result_scope()` uses conservative intersection of registry scopes and result.scope as effective module_scopes. `scope_coverage_report()` returns full dict. CLI (`openamp-foundry simulation-scope-check`) with `--module-id`, `--requested-scopes`, `--format text|json`. `make simulation-scope-check` target. 17 tests. **3255 total.** A simulation module may cover only some biological scopes. If a candidate is evaluated for a scope the module doesn't cover, that result must be flagged as out-of-scope rather than silently trusted. > **New in v0.5.96:** Simulation-module deprecation enforcer (H8) — `DeprecationCheckResult` dataclass (5 fields: module_id, status, is_blocked, block_reason, dry_lab_only). `BLOCKED_STATUSES = {"deprecated", "unavailable"}`. `check_module_deprecation()` looks up module in registry: not-found returns is_blocked=True status="unknown"; deprecated/unavailable returns is_blocked=True with reason; active/experimental returns is_blocked=False. `enforce_deprecation()` filters list[SimulationResult] to only non-blocked modules, returns total_input/passed/blocked/blocked_modules/passed_results/checks/dry_lab_only. `run_deprecation_check_batch()` bulk-checks module_ids with total/blocked/allowed/any_blocked/results/dry_lab_only. CLI (`openamp-foundry simulation-deprecation-check`) with `--module-ids`, `--format text|json`. `make simulation-deprecation-check` target. 21 tests. **3238 total.** Deprecated simulation modules must not be used in production scoring — the enforcer prevents stale or unreliable modules from tainting evidence packets. > **New in v0.5.95:** Simulation-result confidence interval reporter (H7) — `ci_reporter.py` with `ScoreCI` dataclass (9 fields: module_id, score_key, point_estimate, uncertainty, ci_lower, ci_upper, ci_width, overlaps_with, dry_lab_only). `compute_score_ci()` builds CIs from SimulationResult.scores[score_key] ± uncertainty, returns None if score_key missing. `compare_cis()` pairwise checks overlap condition (a_lo <= b_hi and b_lo <= a_hi), returns new list with overlaps_with populated (no in-place mutation). `ci_report()` produces full report with n_results, cis list, any_overlap flag, and dry_lab_only=True. CLI (`openamp-foundry simulation-ci-report`) with `--results-json`, `--score-key`, `--format text|json`. `make simulation-ci-report` target. 16 tests. **3217 total.** Raw scores without uncertainty ranges make it impossible to judge whether two candidates are distinguishable. The CI reporter makes uncertainty explicit and auditable. @@ -88,7 +89,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.97 (H9 — simulation-scope coverage checker) +> **Pipeline version:** v0.5.98 (H10 — simulation-evidence packet assembler) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index c59e7d60..d0f01ae0 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -51,6 +51,68 @@ Honest boundaries: not invalidate results; it highlights them for human review. - All CI reports are dry-lab only and must not be presented as biological proof. +## v0.5.98 — Loop 98: Phase H H10 — Simulation-Evidence Packet Assembler ✓ (2026-07-09) + +`SimulationEvidencePacket` dataclass (12 fields: module_id, result, +requested_scopes, claimed_evidence_level, baseline_beaten, deprecation_check, +scope_check, baseline_check, adapter_gate, effective_evidence_level, +all_checks_passed, failure_reasons, dry_lab_only). +`assemble_evidence_packet()` runs all Phase H sub-checks (deprecation, scope, +baseline, adapter gate) and assembles into a single auditable packet with +all_checks_passed (True only if not deprecation_check["is_blocked"], +scope_check["is_fully_covered"], not baseline_check["capped"], and +adapter_gate["passed"]), failure_reasons (human-readable list of what failed), +and effective_evidence_level (from baseline_check). +`evidence_packet_summary()` returns compact dict with module_id, +claimed_evidence_level, effective_evidence_level, all_checks_passed, +failure_reasons, dry_lab_only. +CLI (`openamp-foundry simulation-evidence-packet`) with `--module-id`, +`--result-json`, `--requested-scopes`, `--claimed-level`, `--baseline-beaten`, +`--format text|json`. `make simulation-evidence-packet` target. +This is the capstone of Phase H — it assembles all the individual simulation +discipline checks into a single auditable evidence packet showing exactly why +a simulation result is trustworthy (or not) enough to support a given evidence +level claim. + +Changes: +- `src/openamp_foundry/simulation/evidence_packet.py` (H10) — Core module + with `SimulationEvidencePacket` dataclass (12+ fields), + `assemble_evidence_packet()` orchestrating all 4 sub-checks with failure + aggregation, `evidence_packet_summary()` compact dict builder. +- `src/openamp_foundry/simulation/__init__.py` — Exports + `SimulationEvidencePacket`, `assemble_evidence_packet`, + `evidence_packet_summary`. +- `src/openamp_foundry/cli/main.py` — Registered `simulation-evidence-packet` + subcommand with `--module-id`, `--result-json`, `--requested-scopes`, + `--claimed-level`, `--baseline-beaten`, `--format` flags. + Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_simulation_evidence_packet()` CLI handler with JSON parsing, + SimulationResult deserialization, text and JSON output, module-id validation. +- `Makefile` — Added `simulation-evidence-packet` target with demo invocation + using membrane_proxy + bacterial_binding + baseline_beaten=false. + Added to `.PHONY`. +- `tests/simulation/test_evidence_packet.py` — 16+ tests covering: returns + packet dataclass, all_checks_passed=true/false for every sub-check, + failure_reasons empty/non-empty, effective_evidence_level capped, + dry_lab_only always True, summary keys and values, deprecated module, + adapter timeout, membrane_proxy passes with correct scope, scope fail. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.98 H10 changelog. Test count: 3271. +- `tests/test_test_count_regression.py` — baseline updated to 3271. + +Honest boundaries: +- The evidence packet aggregates computational sub-checks only. It does not + measure biological activity, safety, or real-world performance. +- `all_checks_passed=True` means all computational discipline checks passed, + not that the simulation result is biologically meaningful. +- The adapter gate uses failure signals provided by the caller; an adapter that + returns plausible-looking but biologically meaningless results will pass. +- Baseline beating is a necessary condition for evidence, not a sufficient one. +- Scope coverage is based on the module registry's declared scopes, which may + be incomplete relative to actual capabilities. +- All evidence packets are dry-lab only and must not be presented as biological + proof. + ## v0.5.97 — Loop 97: Phase H H9 — Simulation-Scope Coverage Checker ✓ (2026-07-09) `ScopeCoverageResult` dataclass (8 fields: module_id, requested_scopes, diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 68ab7111..c4acc32d 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1774,3 +1774,84 @@ def _run_simulation_scope_check(args: argparse.Namespace) -> int: print("not biological proof.") return 0 if result.is_fully_covered else 3 + + +def _run_simulation_evidence_packet(args: argparse.Namespace) -> int: + """Assemble all simulation discipline checks into a single evidence packet.""" + import json + + from openamp_foundry.simulation.evidence_packet import ( + assemble_evidence_packet, + evidence_packet_summary, + ) + from openamp_foundry.simulation.interfaces import SimulationResult + + module_id = args.module_id + output_format = getattr(args, "format", "text") + + try: + raw = json.loads(args.result_json) + except (json.JSONDecodeError, TypeError) as e: + print(json.dumps({ + "status": "error", + "error": f"Invalid JSON for --result-json: {e}", + })) + return 2 + + if not isinstance(raw, dict): + print(json.dumps({ + "status": "error", + "error": "--result-json must be a JSON object (SimulationResult dict)", + })) + return 2 + + if raw.get("module") != module_id: + print(json.dumps({ + "status": "error", + "error": f"--module-id '{module_id}' does not match result.module '{raw.get('module')}'", + })) + return 2 + + result = SimulationResult( + module=raw.get("module", ""), + version=raw.get("version", ""), + scope=raw.get("scope", []), + scores=raw.get("scores", {}), + uncertainty=raw.get("uncertainty", 0.0), + calibration_set=raw.get("calibration_set"), + validated_against=raw.get("validated_against", []), + notes=raw.get("notes", []), + ) + + requested_scopes = [s.strip() for s in args.requested_scopes.split(",") if s.strip()] + claimed_level = args.claimed_level + baseline_beaten = args.baseline_beaten.lower() == "true" + + packet = assemble_evidence_packet( + result=result, + requested_scopes=requested_scopes, + claimed_evidence_level=claimed_level, + baseline_beaten=baseline_beaten, + ) + + summary = evidence_packet_summary(packet) + + if output_format == "json": + print(json.dumps(summary, indent=2)) + else: + status = "PASS" if packet.all_checks_passed else "FAIL" + print(f"Simulation Evidence Packet: {packet.module_id}") + print(f" Status: {status}") + print(f" Claimed evidence level: {packet.claimed_evidence_level}") + print(f" Effective evidence: {packet.effective_evidence_level}") + print(f" All checks passed: {packet.all_checks_passed}") + if packet.failure_reasons: + print(" Failure reasons:") + for reason in packet.failure_reasons: + print(f" - {reason}") + print(f" Dry-lab only: {packet.dry_lab_only}") + print() + print("Dry-lab only. Evidence packets are computational audit artifacts,") + print("not biological proof.") + + return 0 if packet.all_checks_passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 525fa4cd..23dad6f8 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1561,6 +1561,43 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Simulation-evidence packet assembler (H10 — Phase H capstone) ─ + sep = sub.add_parser( + "simulation-evidence-packet", + help=( + "Assemble all simulation discipline checks into a single auditable " + "evidence packet showing why a simulation result is trustworthy " + "enough to support a given evidence level. Dry-lab only." + ), + ) + sep.add_argument( + "--module-id", type=str, required=True, + help="Module ID (validated against result.module).", + ) + sep.add_argument( + "--result-json", type=str, required=True, + help="JSON string of a SimulationResult dict.", + ) + sep.add_argument( + "--requested-scopes", type=str, required=True, + help="Comma-separated scopes (e.g. bacterial_membrane_binding,fungal_binding).", + ) + sep.add_argument( + "--claimed-level", type=int, required=True, + choices=range(1, 7), + help="Claimed evidence level (1-6).", + ) + sep.add_argument( + "--baseline-beaten", type=str, required=True, + choices=["true", "false"], + help="Whether the baseline has been beaten (true or false).", + ) + sep.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Simulation-result confidence interval report (H7) ──────────── sicr = sub.add_parser( "simulation-ci-report", @@ -1766,6 +1803,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "simulation-scope-check": return _run_simulation_scope_check(args) + if args.command == "simulation-evidence-packet": + return _run_simulation_evidence_packet(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index ef2b4c47..7e26d712 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -39,6 +39,11 @@ list_baseline_declarations, validate_baseline_declarations, ) +from .evidence_packet import ( + SimulationEvidencePacket, + assemble_evidence_packet, + evidence_packet_summary, +) from .ci_reporter import ( ScoreCI, ci_report, @@ -69,6 +74,9 @@ from .structure import StructureProxy __all__ = [ + "SimulationEvidencePacket", + "assemble_evidence_packet", + "evidence_packet_summary", "ScopeCoverageResult", "check_scope_coverage", "check_result_scope", diff --git a/src/openamp_foundry/simulation/evidence_packet.py b/src/openamp_foundry/simulation/evidence_packet.py new file mode 100644 index 00000000..a6c432ee --- /dev/null +++ b/src/openamp_foundry/simulation/evidence_packet.py @@ -0,0 +1,128 @@ +"""Simulation-evidence packet assembler — Phase H capstone. + +Assembles all individual simulation discipline checks (baseline beaten, +adapter gate, scope coverage, deprecation, CI, ensemble agreement) into a +single auditable evidence packet showing exactly why a simulation result +is trustworthy (or not) enough to support a given evidence level claim. +""" + +from __future__ import annotations + +from dataclasses import dataclass, asdict +from typing import Any + +from .interfaces import SimulationResult + + +@dataclass +class SimulationEvidencePacket: + module_id: str + result: SimulationResult + requested_scopes: list[str] + claimed_evidence_level: int + baseline_beaten: bool + deprecation_check: dict + scope_check: dict + baseline_check: dict + adapter_gate: dict + effective_evidence_level: int + all_checks_passed: bool + failure_reasons: list[str] + dry_lab_only: bool = True + + +def assemble_evidence_packet( + result: SimulationResult, + requested_scopes: list[str], + claimed_evidence_level: int, + baseline_beaten: bool, + *, + adapter_timeout: bool = False, + adapter_connection_refused: bool = False, + adapter_schema_errors: list[str] | None = None, +) -> SimulationEvidencePacket: + """Run all sub-checks and assemble into a single evidence packet. + + 1. deprecation_check — check_module_deprecation(result.module) + 2. scope_check — check_scope_coverage(result.module, requested_scopes) + 3. baseline_check — check_baseline_requirement(result.module, claimed_evidence_level, baseline_beaten) + 4. adapter_gate — evaluate_adapter_gate(...) + 5. effective_evidence_level — from baseline_check + 6. failure_reasons and all_checks_passed — derived from all sub-checks + """ + from dataclasses import asdict as _asdict + + from .adapter_gate import evaluate_adapter_gate + from .baseline_registry import check_baseline_requirement + from .deprecation_enforcer import check_module_deprecation + from .scope_checker import check_scope_coverage + + deprecation_check = _asdict(check_module_deprecation(result.module)) + scope_check = _asdict(check_scope_coverage(result.module, requested_scopes)) + baseline_check = check_baseline_requirement( + result.module, claimed_evidence_level, baseline_beaten, + ) + + adapter_gate = _asdict(evaluate_adapter_gate( + module_id=result.module, + result=result, + timeout_occurred=adapter_timeout, + connection_refused=adapter_connection_refused, + schema_errors=adapter_schema_errors or [], + module_unavailable=deprecation_check["is_blocked"], + baseline_beaten=baseline_beaten, + )) + + effective_evidence_level = baseline_check["effective_evidence_level"] + + failure_reasons: list[str] = [] + if deprecation_check["is_blocked"]: + failure_reasons.append( + f"Module '{result.module}' is deprecated or unavailable: " + f"{deprecation_check['block_reason']}", + ) + if not scope_check["is_fully_covered"]: + uncovered = scope_check["uncovered"] + failure_reasons.append( + f"Scope(s) not covered: {', '.join(uncovered)}", + ) + if baseline_check["capped"]: + failure_reasons.append( + f"Baseline not beaten — evidence level capped from " + f"{claimed_evidence_level} to {effective_evidence_level}", + ) + if not adapter_gate["passed"]: + reason = adapter_gate["failure_reason"] or "unknown" + detail = adapter_gate["failure_detail"] + failure_reasons.append( + f"Adapter gate failed: {reason} — {detail}", + ) + + all_checks_passed = len(failure_reasons) == 0 + + return SimulationEvidencePacket( + module_id=result.module, + result=result, + requested_scopes=list(requested_scopes), + claimed_evidence_level=claimed_evidence_level, + baseline_beaten=baseline_beaten, + deprecation_check=deprecation_check, + scope_check=scope_check, + baseline_check=baseline_check, + adapter_gate=adapter_gate, + effective_evidence_level=effective_evidence_level, + all_checks_passed=all_checks_passed, + failure_reasons=failure_reasons, + ) + + +def evidence_packet_summary(packet: SimulationEvidencePacket) -> dict[str, Any]: + """Return a compact summary dict.""" + return { + "module_id": packet.module_id, + "claimed_evidence_level": packet.claimed_evidence_level, + "effective_evidence_level": packet.effective_evidence_level, + "all_checks_passed": packet.all_checks_passed, + "failure_reasons": packet.failure_reasons, + "dry_lab_only": packet.dry_lab_only, + } diff --git a/tests/simulation/test_evidence_packet.py b/tests/simulation/test_evidence_packet.py new file mode 100644 index 00000000..1f72bf80 --- /dev/null +++ b/tests/simulation/test_evidence_packet.py @@ -0,0 +1,272 @@ +"""Tests for the simulation-evidence packet assembler (Phase H H10).""" + +from __future__ import annotations + +from dataclasses import asdict + +import pytest + +from openamp_foundry.simulation.evidence_packet import ( + SimulationEvidencePacket, + assemble_evidence_packet, + evidence_packet_summary, +) +from openamp_foundry.simulation.interfaces import SimulationResult + + +def _make_result( + module: str = "membrane_proxy", + scope: list[str] | None = None, +) -> SimulationResult: + return SimulationResult( + module=module, + version="0.1.0", + scope=scope or ["bacterial_membrane_binding"], + scores={"binding_energy": 0.75}, + uncertainty=0.1, + calibration_set=None, + validated_against=["APD3"], + notes=[], + ) + + +# ── assemble_evidence_packet returns SimulationEvidencePacket ────── + +def test_assemble_returns_packet(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + ) + assert isinstance(packet, SimulationEvidencePacket) + + +# ── all_checks_passed=True when all sub-checks pass ──────────────── + +def test_all_checks_passed_all_green(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + ) + assert packet.all_checks_passed is True + assert packet.failure_reasons == [] + + +# ── all_checks_passed=False when deprecation blocked ─────────────── + +def test_deprecation_blocked_fails(): + result = _make_result(module="dummy_membrane_proxy") + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=1, + baseline_beaten=True, + ) + assert packet.all_checks_passed is False + assert any("deprecated" in r.lower() for r in packet.failure_reasons) + + +# ── all_checks_passed=False when scope uncovered ─────────────────── + +def test_scope_uncovered_fails(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["fungal_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + ) + assert packet.all_checks_passed is False + assert any("scope" in r.lower() for r in packet.failure_reasons) + + +# ── all_checks_passed=False when baseline capped ─────────────────── + +def test_baseline_capped_fails(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=False, + ) + assert packet.all_checks_passed is False + assert any("baseline" in r.lower() for r in packet.failure_reasons) + + +# ── all_checks_passed=False when adapter gate fails ──────────────── + +def test_adapter_timeout_fails(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + adapter_timeout=True, + ) + assert packet.all_checks_passed is False + assert any("timeout" in r.lower() for r in packet.failure_reasons) + + +# ── failure_reasons non-empty when any check fails ───────────────── + +def test_failure_reasons_non_empty_on_failure(): + result = _make_result(module="dummy_membrane_proxy") + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=False, + ) + assert len(packet.failure_reasons) > 0 + + +# ── failure_reasons empty when all pass ──────────────────────────── + +def test_failure_reasons_empty_on_pass(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + ) + assert packet.failure_reasons == [] + + +# ── effective_evidence_level capped when baseline not beaten ─────── + +def test_effective_level_capped_when_baseline_not_beaten(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=False, + ) + assert packet.effective_evidence_level < packet.claimed_evidence_level + + +# ── dry_lab_only always True ─────────────────────────────────────── + +def test_dry_lab_only_always_true(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + ) + assert packet.dry_lab_only is True + + +# ── evidence_packet_summary returns correct keys ─────────────────── + +def test_summary_has_correct_keys(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + ) + summary = evidence_packet_summary(packet) + expected_keys = { + "module_id", "claimed_evidence_level", "effective_evidence_level", + "all_checks_passed", "failure_reasons", "dry_lab_only", + } + assert set(summary.keys()) == expected_keys + + +# ── evidence_packet_summary dry_lab_only=True ────────────────────── + +def test_summary_dry_lab_only_true(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + ) + summary = evidence_packet_summary(packet) + assert summary["dry_lab_only"] is True + + +# ── deprecated module → all_checks_passed=False ──────────────────── + +def test_deprecated_module_all_checks_false(): + result = _make_result(module="dummy_membrane_proxy") + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=1, + baseline_beaten=True, + ) + assert packet.all_checks_passed is False + + +# ── adapter timeout → all_checks_passed=False ────────────────────── + +def test_adapter_timeout_all_checks_false(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + adapter_timeout=True, + ) + assert packet.all_checks_passed is False + + +# ── membrane_proxy + correct scope + baseline_beaten=True → pass ─── + +def test_membrane_proxy_bacterial_pass(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + ) + assert packet.all_checks_passed is True + assert packet.effective_evidence_level == 2 + + +# ── membrane_proxy + fungal_binding → scope uncovered fail ───────── + +def test_membrane_proxy_fungal_scope_fails(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["fungal_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + ) + assert packet.all_checks_passed is False + assert any("scope" in r.lower() for r in packet.failure_reasons) + + +# ── assemble_evidence_packet stores all fields correctly ─────────── + +def test_packet_stores_all_fields(): + result = _make_result() + packet = assemble_evidence_packet( + result=result, + requested_scopes=["bacterial_membrane_binding"], + claimed_evidence_level=2, + baseline_beaten=True, + ) + assert packet.module_id == "membrane_proxy" + assert packet.claimed_evidence_level == 2 + assert packet.baseline_beaten is True + assert isinstance(packet.deprecation_check, dict) + assert isinstance(packet.scope_check, dict) + assert isinstance(packet.baseline_check, dict) + assert isinstance(packet.adapter_gate, dict) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 6660eff7..ba7978cd 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3255 tests as of Loop 97 (2026-07-09) -- Phase H H9 (simulation-scope coverage checker) +# Baseline: 3272 tests as of Loop 98 (2026-07-09) -- Phase H H10 (simulation-evidence packet assembler) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3255 +BASELINE = 3272 TOLERANCE = 0.05 # allow 5% fluctuation