From c63c6f96dcb224c2292a1dcecdc3c69d872789c5 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 13:50:03 +0700 Subject: [PATCH] feat: Phase H H7 simulation-result confidence interval reporter -- uncertainty is explicit and auditable --- Makefile | 7 +- docs/evidence/METRICS_CURRENT.md | 5 +- docs/research/ROADMAP.md | 51 ++++ src/openamp_foundry/cli/commands/reports.py | 64 +++++ src/openamp_foundry/cli/main.py | 28 +- src/openamp_foundry/simulation/__init__.py | 10 + src/openamp_foundry/simulation/ci_reporter.py | 120 +++++++++ tests/simulation/test_ci_reporter.py | 252 ++++++++++++++++++ tests/test_test_count_regression.py | 4 +- 9 files changed, 535 insertions(+), 6 deletions(-) create mode 100644 src/openamp_foundry/simulation/ci_reporter.py create mode 100644 tests/simulation/test_ci_reporter.py diff --git a/Makefile b/Makefile index b6a85751..67c93222 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -751,3 +751,8 @@ simulation-ensemble-check: --sequence AKLWKR \ --results-json '[{"module":"membrane_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.75},"uncertainty":0.1,"calibration_set":null,"validated_against":["APD3"],"notes":[]},{"module":"structure_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.80},"uncertainty":0.1,"calibration_set":null,"validated_against":["APD3"],"notes":[]}]' @echo "Ensemble check complete." + +simulation-ci-report: + .venv/bin/openamp-foundry simulation-ci-report \ + --results-json '[{"module":"membrane_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.75},"uncertainty":0.1,"calibration_set":null,"validated_against":["APD3"],"notes":[]},{"module":"structure_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.80},"uncertainty":0.15,"calibration_set":null,"validated_against":["APD3"],"notes":[]}]' + @echo "CI report complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 8c738ca5..40b0a659 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase H H6 — simulation-ensemble agreement checker — v0.5.94) +> **Last updated:** 2026-07-09 (Phase H H7 — simulation-result confidence interval reporter — v0.5.95) +> **New in v0.5.95:** Simulation-result confidence interval reporter (H7) — `ci_reporter.py` with `ScoreCI` dataclass (9 fields: module_id, score_key, point_estimate, uncertainty, ci_lower, ci_upper, ci_width, overlaps_with, dry_lab_only). `compute_score_ci()` builds CIs from SimulationResult.scores[score_key] ± uncertainty, returns None if score_key missing. `compare_cis()` pairwise checks overlap condition (a_lo <= b_hi and b_lo <= a_hi), returns new list with overlaps_with populated (no in-place mutation). `ci_report()` produces full report with n_results, cis list, any_overlap flag, and dry_lab_only=True. CLI (`openamp-foundry simulation-ci-report`) with `--results-json`, `--score-key`, `--format text|json`. `make simulation-ci-report` target. 16 tests. **3217 total.** Raw scores without uncertainty ranges make it impossible to judge whether two candidates are distinguishable. The CI reporter makes uncertainty explicit and auditable. > **New in v0.5.94:** Simulation-ensemble agreement checker (H6) — `ensemble_checker.py` with `EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked, agreement_level, agreement_description, mean_score, score_range, scores_by_module, threshold, dry_lab_only). `AGREEMENT_LEVELS` dict (5 levels: strong, moderate, weak, conflict, insufficient). `check_ensemble_agreement()` extracts score_key from each SimulationResult's scores dict, computes score_range, classifies into strong (≥3 modules within threshold), moderate (2 modules within threshold), weak (1 module), conflict (beyond threshold), or insufficient (no results). `run_ensemble_check_batch()` aggregates multiple calls with counts and any_conflict flag. CLI (`openamp-foundry simulation-ensemble-check`) with `--sequence`, `--results-json`, `--score-key`, `--threshold`, `--format text|json`. `make simulation-ensemble-check` target. 20 tests. **3201 total.** When multiple simulation modules independently agree on a candidate, that agreement is stronger evidence than a single module alone. The ensemble checker makes this agreement explicit and auditable. > **New in v0.5.93:** Simulation-result provenance chain (H5) — `SimulationProvenanceRecord` dataclass with run_id, module_id, module_version, timestamp_utc, input_hash (SHA-256 of input sequence), result_hash (SHA-256 of sorted scores dict), calibration_set, notes, and dry_lab_only. `make_provenance_record()` computes input_hash and result_hash deterministically (sort_keys=True for result_hash). `validate_provenance_record()` checks run_id, module_id, module_version non-empty; ISO 8601 timestamp contains 'T'; hashes are 64-char hex; dry_lab_only is True. `provenance_summary()` aggregates total, unique modules, run_ids, and dry_lab_only flag. CLI (`openamp-foundry simulation-provenance`) with `--run-id`, `--module-id`, `--module-version`, `--timestamp-utc`, `--input-sequence`, `--scores-json`, `--calibration-set`, `--format text|json`. `make simulation-provenance` target with demo invocation. 19 tests. **3181 total.** Every simulation result carries a traceable provenance chain so results can be audited, reproduced, or invalidated later without relying on memory. > **New in v0.5.92:** Fail-closed adapter integration tests (H4) — `FAIL_CLOSED_REASONS` dict keys (6 entries: timeout, connection_refused, invalid_response, schema_violation, module_unavailable, baseline_not_beaten). `AdapterGateResult` dataclass with module_id, passed, failure_reason, failure_detail, dry_lab_only. `evaluate_adapter_gate()` fail-closed: returns passed=False on ANY failure signal with priority ordering (timeout > connection_refused > invalid_response > schema_violation > module_unavailable > baseline_not_beaten). `run_adapter_gate_batch()` aggregates multiple adapter calls with total/passed/failed/any_failed/results/dry_lab_only. CLI (`openamp-foundry adapter-gate-check`) with `--module-id`, `--timeout`, `--connection-refused`, `--schema-errors`, `--module-unavailable`, `--baseline-beaten`, `--format text|json`. `make adapter-gate-check` target. 21 tests. **3162 total.** Avoids hidden external failures — when the adapter to an external simulation service is down or misbehaves, the pipeline must fail loudly rather than silently passing garbage through. @@ -85,7 +86,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.94 (H6 — simulation-ensemble agreement checker) +> **Pipeline version:** v0.5.95 (H7 — simulation-result confidence interval reporter) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 527db284..c119526f 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,56 @@ # Roadmap +## v0.5.95 — Loop 95: Phase H H7 — Simulation-Result Confidence Interval Reporter ✓ (2026-07-09) + +`ScoreCI` dataclass (9 fields: module_id, score_key, point_estimate, uncertainty, +ci_lower, ci_upper, ci_width, overlaps_with, dry_lab_only). +`compute_score_ci()` builds CIs from SimulationResult.scores[score_key] ± uncertainty, +returns None if score_key missing. `compare_cis()` pairwise checks overlap condition +(a_lo <= b_hi and b_lo <= a_hi), returns new list with overlaps_with populated +(no in-place mutation). `ci_report()` produces full report with n_results, cis list, +any_overlap flag, and dry_lab_only=True. +CLI (`openamp-foundry simulation-ci-report`) with `--results-json`, `--score-key`, +`--format text|json`. `make simulation-ci-report` target with demo invocation. +Raw scores without uncertainty ranges make it impossible to judge whether two +candidates are distinguishable. The CI reporter makes uncertainty explicit and +auditable. + +Changes: +- `src/openamp_foundry/simulation/ci_reporter.py` (H7) — Core module with + `ScoreCI` dataclass (9 fields), `compute_score_ci()` with CI bounds from + score ± uncertainty, `compare_cis()` with pairwise overlap detection, + `ci_report()` with n_results, any_overlap, and dry_lab_only=True. +- `src/openamp_foundry/simulation/__init__.py` — Exports `ScoreCI`, + `compute_score_ci`, `compare_cis`, `ci_report`. +- `src/openamp_foundry/cli/main.py` — Registered `simulation-ci-report` + subcommand with `--results-json`, `--score-key`, `--format` flags. + Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_simulation_ci_report()` CLI handler with JSON parsing, + SimulationResult deserialization, text and JSON output. +- `Makefile` — Added `simulation-ci-report` target with demo invocation + using membrane_proxy + structure_proxy. Added to `.PHONY`. +- `tests/simulation/test_ci_reporter.py` — 16 tests covering: compute_score_ci + returns correct bounds, ci_width = 2*uncertainty, missing key returns None, + dry_lab_only always True, non-overlapping CIs empty overlaps_with, + overlapping CIs populated overlaps_with, no in-place mutation, 0 results, + n_results correct, dry_lab_only=True, any_overlap false/true, overlapping + example, non-overlapping example. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.95 H7 changelog. Test count: 3217. +- `tests/test_test_count_regression.py` — baseline updated to 3217. + +Honest boundaries: +- Confidence intervals are computed from SimulationResult.uncertainty, which + is self-reported by each simulation module and not independently verified. +- Overlap detection is a mathematical condition: two CIs overlapping means + the scores are not statistically distinguishable at the stated uncertainty + level. It does not mean the modules are biologically correct or incorrect. +- A CI with no overlaps may still be biologically meaningless — precision + without accuracy is not useful. +- The any_overlap flag is a descriptive summary, not a gate. Overlap does + not invalidate results; it highlights them for human review. +- All CI reports are dry-lab only and must not be presented as biological proof. + ## v0.5.94 — Loop 94: Phase H H6 — Simulation-Ensemble Agreement Checker ✓ (2026-07-09) `EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked, diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index b4f5a435..be518788 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1568,6 +1568,70 @@ def _run_adapter_gate_check(args: argparse.Namespace) -> int: return 0 if gate_result.passed else 3 +def _run_simulation_ci_report(args: argparse.Namespace) -> int: + """Compute confidence intervals and overlap report for simulation results.""" + import json + + from openamp_foundry.simulation.ci_reporter import ci_report + from openamp_foundry.simulation.interfaces import SimulationResult + + output_format = getattr(args, "format", "text") + score_key = args.score_key + + try: + raw_results = json.loads(args.results_json) + except (json.JSONDecodeError, TypeError) as e: + print(json.dumps({ + "status": "error", + "error": f"Invalid JSON for --results-json: {e}", + })) + return 2 + + if not isinstance(raw_results, list): + print(json.dumps({ + "status": "error", + "error": "--results-json must be a JSON array of SimulationResult dicts", + })) + return 2 + + results = [] + for item in raw_results: + results.append(SimulationResult( + module=item.get("module", ""), + version=item.get("version", ""), + scope=item.get("scope", []), + scores=item.get("scores", {}), + uncertainty=item.get("uncertainty", 0.0), + calibration_set=item.get("calibration_set"), + validated_against=item.get("validated_against", []), + notes=item.get("notes", []), + )) + + report = ci_report(results, score_key=score_key) + + if output_format == "json": + print(json.dumps(report, indent=2)) + else: + print(f"Simulation Confidence Interval Report") + print(f" Score key: {report['score_key']}") + print(f" N results: {report['n_results']}") + print(f" Any overlap: {report['any_overlap']}") + print() + for ci_dict in report["cis"]: + ov = ", ".join(ci_dict["overlaps_with"]) if ci_dict["overlaps_with"] else "(none)" + print(f" [{ci_dict['module_id']}]") + print(f" Point estimate: {ci_dict['point_estimate']:.4f}") + print(f" Uncertainty: {ci_dict['uncertainty']:.4f}") + print(f" CI: [{ci_dict['ci_lower']:.4f}, {ci_dict['ci_upper']:.4f}]") + print(f" Width: {ci_dict['ci_width']:.4f}") + print(f" Overlaps with: {ov}") + print() + print("Dry-lab only. Confidence intervals are computational estimates,") + print("not biological proof.") + + return 0 + + def _run_simulation_provenance(args: argparse.Namespace) -> int: """Generate and display a simulation-result provenance record.""" import json diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 906d0246..43dbd378 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1519,6 +1519,29 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Simulation-result confidence interval report (H7) ──────────── + sicr = sub.add_parser( + "simulation-ci-report", + help=( + "Compute confidence intervals from SimulationResult uncertainty fields " + "and compare them for overlap. Makes uncertainty explicit and auditable. " + "Dry-lab only." + ), + ) + sicr.add_argument( + "--results-json", type=str, required=True, + help="JSON array of SimulationResult dicts.", + ) + sicr.add_argument( + "--score-key", type=str, default="binding_energy", + help="Score key to extract from each result's scores dict (default: binding_energy).", + ) + sicr.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + return parser @@ -1692,6 +1715,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "simulation-ensemble-check": return _run_simulation_ensemble_check(args) + if args.command == "simulation-ci-report": + return _run_simulation_ci_report(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index 96662a70..9db970ab 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -26,6 +26,12 @@ list_baseline_declarations, validate_baseline_declarations, ) +from .ci_reporter import ( + ScoreCI, + ci_report, + compare_cis, + compute_score_ci, +) from .gate import SimulationGateVerdict, evaluate_simulation_gate from .interfaces import ( EmulatorBaseline, @@ -58,6 +64,10 @@ "EnsembleAgreementResult", "check_ensemble_agreement", "run_ensemble_check_batch", + "ScoreCI", + "compute_score_ci", + "compare_cis", + "ci_report", "SimulationResult", "VirtualAssayProxy", "ExternalSimulationAdapter", diff --git a/src/openamp_foundry/simulation/ci_reporter.py b/src/openamp_foundry/simulation/ci_reporter.py new file mode 100644 index 00000000..fa7b5279 --- /dev/null +++ b/src/openamp_foundry/simulation/ci_reporter.py @@ -0,0 +1,120 @@ +"""Confidence interval reporter for simulation results. + +Raw scores without uncertainty ranges make it impossible to judge +whether two candidates are distinguishable. A CI reporter makes the +uncertainty explicit and auditable. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, asdict +from typing import Any + +from .interfaces import SimulationResult + + +@dataclass +class ScoreCI: + module_id: str + score_key: str + point_estimate: float + uncertainty: float + ci_lower: float + ci_upper: float + ci_width: float + overlaps_with: list[str] = field(default_factory=list) + dry_lab_only: bool = True + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def compute_score_ci( + result: SimulationResult, + score_key: str = "binding_energy", +) -> ScoreCI | None: + """Return ScoreCI for the given score_key, or None if score_key missing. + + ci_lower = score - uncertainty, ci_upper = score + uncertainty. + overlaps_with starts as empty list (populated by compare_cis). + """ + if score_key not in result.scores: + return None + + point_estimate = result.scores[score_key] + uncertainty = result.uncertainty + ci_lower = point_estimate - uncertainty + ci_upper = point_estimate + uncertainty + ci_width = ci_upper - ci_lower + + return ScoreCI( + module_id=result.module, + score_key=score_key, + point_estimate=point_estimate, + uncertainty=uncertainty, + ci_lower=ci_lower, + ci_upper=ci_upper, + ci_width=ci_width, + ) + + +def compare_cis( + cis: list[ScoreCI], +) -> list[ScoreCI]: + """For each CI, populate overlaps_with with module_ids whose CIs overlap. + + Two CIs [a_lo, a_hi] and [b_lo, b_hi] overlap iff a_lo <= b_hi and b_lo <= a_hi. + Returns new list of ScoreCI with overlaps_with populated (no in-place mutation). + """ + result: list[ScoreCI] = [] + for i, ci_a in enumerate(cis): + overlapping_ids: list[str] = [] + for j, ci_b in enumerate(cis): + if i == j: + continue + if ci_a.ci_lower <= ci_b.ci_upper and ci_b.ci_lower <= ci_a.ci_upper: + overlapping_ids.append(ci_b.module_id) + result.append(ScoreCI( + module_id=ci_a.module_id, + score_key=ci_a.score_key, + point_estimate=ci_a.point_estimate, + uncertainty=ci_a.uncertainty, + ci_lower=ci_a.ci_lower, + ci_upper=ci_a.ci_upper, + ci_width=ci_a.ci_width, + overlaps_with=overlapping_ids, + dry_lab_only=True, + )) + return result + + +def ci_report( + results: list[SimulationResult], + score_key: str = "binding_energy", +) -> dict: + """Compute CIs for all results and compare them. + + Returns: + - "score_key": str + - "n_results": int + - "cis": list of ScoreCI as dicts (with overlaps_with populated) + - "any_overlap": bool (True if any CI has at least one overlap) + - "dry_lab_only": True + """ + score_cis: list[ScoreCI] = [] + for result in results: + ci = compute_score_ci(result, score_key=score_key) + if ci is not None: + score_cis.append(ci) + + compared = compare_cis(score_cis) if score_cis else [] + + any_overlap = any(len(c.overlaps_with) > 0 for c in compared) + + return { + "score_key": score_key, + "n_results": len(results), + "cis": [c.to_dict() for c in compared], + "any_overlap": any_overlap, + "dry_lab_only": True, + } diff --git a/tests/simulation/test_ci_reporter.py b/tests/simulation/test_ci_reporter.py new file mode 100644 index 00000000..90491fc2 --- /dev/null +++ b/tests/simulation/test_ci_reporter.py @@ -0,0 +1,252 @@ +"""Tests for simulation confidence interval reporter.""" +from __future__ import annotations + +import pytest + +from openamp_foundry.simulation.ci_reporter import ( + ScoreCI, + ci_report, + compare_cis, + compute_score_ci, +) +from openamp_foundry.simulation.interfaces import SimulationResult + + +def _make_result( + module: str, + score: float, + uncertainty: float = 0.1, + score_key: str = "binding_energy", +) -> SimulationResult: + return SimulationResult( + module=module, + version="0.1.0", + scope=["bacterial_binding"], + scores={score_key: score}, + uncertainty=uncertainty, + calibration_set=None, + validated_against=["APD3"], + notes=[], + ) + + +class TestComputeScoreCI: + def test_returns_score_ci_with_correct_bounds(self): + result = _make_result("membrane_proxy", 0.75, uncertainty=0.1) + ci = compute_score_ci(result) + assert ci is not None + assert ci.module_id == "membrane_proxy" + assert ci.score_key == "binding_energy" + assert ci.point_estimate == 0.75 + assert ci.uncertainty == 0.1 + assert ci.ci_lower == 0.65 + assert ci.ci_upper == 0.85 + + def test_ci_width_equals_two_times_uncertainty(self): + result = _make_result("membrane_proxy", 0.75, uncertainty=0.1) + ci = compute_score_ci(result) + assert ci is not None + assert ci.ci_width == pytest.approx(0.20) + assert ci.ci_width == pytest.approx(2 * ci.uncertainty) + + def test_missing_score_key_returns_none(self): + result = _make_result("membrane_proxy", 0.75, score_key="binding_energy") + ci = compute_score_ci(result, score_key="other_key") + assert ci is None + + def test_dry_lab_only_always_true(self): + result = _make_result("membrane_proxy", 0.75) + ci = compute_score_ci(result) + assert ci is not None + assert ci.dry_lab_only is True + + +class TestCompareCIs: + def test_non_overlapping_cis_have_empty_overlaps_with(self): + cis = [ + ScoreCI( + module_id="module_a", + score_key="binding_energy", + point_estimate=0.2, + uncertainty=0.05, + ci_lower=0.15, + ci_upper=0.25, + ci_width=0.10, + ), + ScoreCI( + module_id="module_b", + score_key="binding_energy", + point_estimate=0.9, + uncertainty=0.05, + ci_lower=0.85, + ci_upper=0.95, + ci_width=0.10, + ), + ] + result = compare_cis(cis) + assert result[0].overlaps_with == [] + assert result[1].overlaps_with == [] + + def test_overlapping_cis_have_populated_overlaps_with(self): + cis = [ + ScoreCI( + module_id="module_a", + score_key="binding_energy", + point_estimate=0.5, + uncertainty=0.3, + ci_lower=0.2, + ci_upper=0.8, + ci_width=0.60, + ), + ScoreCI( + module_id="module_b", + score_key="binding_energy", + point_estimate=0.6, + uncertainty=0.3, + ci_lower=0.3, + ci_upper=0.9, + ci_width=0.60, + ), + ] + result = compare_cis(cis) + assert "module_b" in result[0].overlaps_with + assert "module_a" in result[1].overlaps_with + + def test_does_not_mutate_input(self): + cis = [ + ScoreCI( + module_id="module_a", + score_key="binding_energy", + point_estimate=0.5, + uncertainty=0.3, + ci_lower=0.2, + ci_upper=0.8, + ci_width=0.60, + ), + ScoreCI( + module_id="module_b", + score_key="binding_energy", + point_estimate=0.6, + uncertainty=0.3, + ci_lower=0.3, + ci_upper=0.9, + ci_width=0.60, + ), + ] + original_ids = [c.module_id for c in cis] + compare_cis(cis) + assert [c.overlaps_with for c in cis] == [[], []] + assert [c.module_id for c in cis] == original_ids + + def test_overlap_example_half_plus_minus_three(self): + cis = [ + ScoreCI( + module_id="module_a", + score_key="binding_energy", + point_estimate=0.5, + uncertainty=0.3, + ci_lower=0.2, + ci_upper=0.8, + ci_width=0.60, + ), + ScoreCI( + module_id="module_b", + score_key="binding_energy", + point_estimate=0.6, + uncertainty=0.3, + ci_lower=0.3, + ci_upper=0.9, + ci_width=0.60, + ), + ] + result = compare_cis(cis) + assert len(result[0].overlaps_with) == 1 + assert len(result[1].overlaps_with) == 1 + + def test_no_overlap_example_narrow_cis(self): + cis = [ + ScoreCI( + module_id="module_a", + score_key="binding_energy", + point_estimate=0.2, + uncertainty=0.05, + ci_lower=0.15, + ci_upper=0.25, + ci_width=0.10, + ), + ScoreCI( + module_id="module_b", + score_key="binding_energy", + point_estimate=0.9, + uncertainty=0.05, + ci_lower=0.85, + ci_upper=0.95, + ci_width=0.10, + ), + ] + result = compare_cis(cis) + assert len(result[0].overlaps_with) == 0 + assert len(result[1].overlaps_with) == 0 + + +class TestCIReport: + def test_empty_results(self): + report = ci_report([], score_key="binding_energy") + assert report["n_results"] == 0 + assert report["cis"] == [] + assert report["any_overlap"] is False + + def test_n_results_correct(self): + results = [ + _make_result("module_a", 0.75), + _make_result("module_b", 0.80), + ] + report = ci_report(results) + assert report["n_results"] == 2 + + def test_dry_lab_only_true(self): + results = [ + _make_result("module_a", 0.75), + ] + report = ci_report(results) + assert report["dry_lab_only"] is True + + def test_any_overlap_false_when_no_overlaps(self): + results = [ + _make_result("module_a", 0.2, uncertainty=0.05), + _make_result("module_b", 0.9, uncertainty=0.05), + ] + report = ci_report(results) + assert report["any_overlap"] is False + + def test_any_overlap_true_when_overlaps_exist(self): + results = [ + _make_result("module_a", 0.5, uncertainty=0.3), + _make_result("module_b", 0.6, uncertainty=0.3), + ] + report = ci_report(results) + assert report["any_overlap"] is True + + def test_overlapping_example(self): + results = [ + _make_result("module_a", 0.5, uncertainty=0.3), + _make_result("module_b", 0.6, uncertainty=0.3), + ] + report = ci_report(results) + assert report["any_overlap"] is True + ci_a = [c for c in report["cis"] if c["module_id"] == "module_a"][0] + ci_b = [c for c in report["cis"] if c["module_id"] == "module_b"][0] + assert "module_b" in ci_a["overlaps_with"] + assert "module_a" in ci_b["overlaps_with"] + + def test_non_overlapping_example(self): + results = [ + _make_result("module_a", 0.2, uncertainty=0.05), + _make_result("module_b", 0.9, uncertainty=0.05), + ] + report = ci_report(results) + assert report["any_overlap"] is False + ci_a = [c for c in report["cis"] if c["module_id"] == "module_a"][0] + ci_b = [c for c in report["cis"] if c["module_id"] == "module_b"][0] + assert ci_a["overlaps_with"] == [] + assert ci_b["overlaps_with"] == [] diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index b120abf2..cb0f257f 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3201 tests as of Loop 94 (2026-07-09) -- Phase H H6 (simulation-ensemble agreement checker) +# Baseline: 3217 tests as of Loop 95 (2026-07-09) -- Phase H H7 (simulation-result confidence interval reporter) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3201 +BASELINE = 3217 TOLERANCE = 0.05 # allow 5% fluctuation