diff --git a/Makefile b/Makefile index c002b2fe..b6a85751 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -745,3 +745,9 @@ simulation-provenance: --input-sequence AKLWKR \ --scores-json '{"binding_energy": 0.75}' @echo "Provenance record complete." + +simulation-ensemble-check: + .venv/bin/openamp-foundry simulation-ensemble-check \ + --sequence AKLWKR \ + --results-json '[{"module":"membrane_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.75},"uncertainty":0.1,"calibration_set":null,"validated_against":["APD3"],"notes":[]},{"module":"structure_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.80},"uncertainty":0.1,"calibration_set":null,"validated_against":["APD3"],"notes":[]}]' + @echo "Ensemble check complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 35a43558..8c738ca5 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase H H5 — simulation-result provenance chain — v0.5.93) +> **Last updated:** 2026-07-09 (Phase H H6 — simulation-ensemble agreement checker — v0.5.94) +> **New in v0.5.94:** Simulation-ensemble agreement checker (H6) — `ensemble_checker.py` with `EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked, agreement_level, agreement_description, mean_score, score_range, scores_by_module, threshold, dry_lab_only). `AGREEMENT_LEVELS` dict (5 levels: strong, moderate, weak, conflict, insufficient). `check_ensemble_agreement()` extracts score_key from each SimulationResult's scores dict, computes score_range, classifies into strong (≥3 modules within threshold), moderate (2 modules within threshold), weak (1 module), conflict (beyond threshold), or insufficient (no results). `run_ensemble_check_batch()` aggregates multiple calls with counts and any_conflict flag. CLI (`openamp-foundry simulation-ensemble-check`) with `--sequence`, `--results-json`, `--score-key`, `--threshold`, `--format text|json`. `make simulation-ensemble-check` target. 20 tests. **3201 total.** When multiple simulation modules independently agree on a candidate, that agreement is stronger evidence than a single module alone. The ensemble checker makes this agreement explicit and auditable. > **New in v0.5.93:** Simulation-result provenance chain (H5) — `SimulationProvenanceRecord` dataclass with run_id, module_id, module_version, timestamp_utc, input_hash (SHA-256 of input sequence), result_hash (SHA-256 of sorted scores dict), calibration_set, notes, and dry_lab_only. `make_provenance_record()` computes input_hash and result_hash deterministically (sort_keys=True for result_hash). `validate_provenance_record()` checks run_id, module_id, module_version non-empty; ISO 8601 timestamp contains 'T'; hashes are 64-char hex; dry_lab_only is True. `provenance_summary()` aggregates total, unique modules, run_ids, and dry_lab_only flag. CLI (`openamp-foundry simulation-provenance`) with `--run-id`, `--module-id`, `--module-version`, `--timestamp-utc`, `--input-sequence`, `--scores-json`, `--calibration-set`, `--format text|json`. `make simulation-provenance` target with demo invocation. 19 tests. **3181 total.** Every simulation result carries a traceable provenance chain so results can be audited, reproduced, or invalidated later without relying on memory. > **New in v0.5.92:** Fail-closed adapter integration tests (H4) — `FAIL_CLOSED_REASONS` dict keys (6 entries: timeout, connection_refused, invalid_response, schema_violation, module_unavailable, baseline_not_beaten). `AdapterGateResult` dataclass with module_id, passed, failure_reason, failure_detail, dry_lab_only. `evaluate_adapter_gate()` fail-closed: returns passed=False on ANY failure signal with priority ordering (timeout > connection_refused > invalid_response > schema_violation > module_unavailable > baseline_not_beaten). `run_adapter_gate_batch()` aggregates multiple adapter calls with total/passed/failed/any_failed/results/dry_lab_only. CLI (`openamp-foundry adapter-gate-check`) with `--module-id`, `--timeout`, `--connection-refused`, `--schema-errors`, `--module-unavailable`, `--baseline-beaten`, `--format text|json`. `make adapter-gate-check` target. 21 tests. **3162 total.** Avoids hidden external failures — when the adapter to an external simulation service is down or misbehaves, the pipeline must fail loudly rather than silently passing garbage through. > **New in v0.5.91:** Per-module cheapest-baseline declaration (H3) — `BaselineDeclaration` dataclass with module_id, module_name, baseline_description, baseline_type, evidence_level_ceiling, and notes. `BASELINE_DECLARATIONS` list (4 entries: membrane_proxy, structure_proxy, dummy_membrane_proxy, external_adapter_placeholder). `get_baseline_declaration()` and `list_baseline_declarations()` for lookup. `check_baseline_requirement()` caps effective_evidence_level to ceiling when baseline not beaten; returns dict with module_id, baseline_beaten, claimed/effective evidence levels, capped flag, message, and dry_lab_only. `validate_baseline_declarations()` checks module_id non-empty, baseline_description non-empty, baseline_type in {heuristic, random, constant, length}, evidence_level_ceiling 1-6, no duplicate module_ids. CLI (`openamp-foundry simulation-baseline-check`) with `--module-id`, `--claimed-level`, `--baseline-beaten`, `--format text|json`. `make simulation-baseline-check` target. 16 tests. **3141 total.** Forces honest enemy comparison — every simulation module must declare the simplest baseline it must beat. @@ -84,7 +85,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.93 (H5 — simulation-result provenance chain) +> **Pipeline version:** v0.5.94 (H6 — simulation-ensemble agreement checker) > **Branch:** main --- @@ -1154,7 +1155,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 3162 | +| Total tests | 3201 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring modules | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 8ac0e3b8..527db284 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,59 @@ # Roadmap +## v0.5.94 — Loop 94: Phase H H6 — Simulation-Ensemble Agreement Checker ✓ (2026-07-09) + +`EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked, +agreement_level, agreement_description, mean_score, score_range, scores_by_module, +threshold, dry_lab_only). `AGREEMENT_LEVELS` dict (5 levels: strong/moderate/ +weak/conflict/insufficient). `check_ensemble_agreement()` extracts score_key from +each SimulationResult's scores dict, computes score_range, classifies agreement: +strong (≥3 modules within threshold), moderate (2 modules within threshold), +weak (1 module), conflict (beyond threshold), or insufficient (no results). +`run_ensemble_check_batch()` aggregates multiple calls with counts and any_conflict +flag. CLI (`openamp-foundry simulation-ensemble-check`) with `--sequence`, +`--results-json`, `--score-key`, `--threshold`, `--format text|json`. +`make simulation-ensemble-check` target with demo invocation. When multiple +simulation modules independently agree on a candidate, that agreement is stronger +evidence than a single module alone. The ensemble checker makes this agreement +explicit and auditable. + +Changes: +- `src/openamp_foundry/simulation/ensemble_checker.py` (H6) — Core module with + `AGREEMENT_LEVELS` dict (5 entries), `EnsembleAgreementResult` dataclass + (9 fields), `check_ensemble_agreement()` with 5-path classification logic, + `run_ensemble_check_batch()` with per-level counts and any_conflict flag. +- `src/openamp_foundry/simulation/__init__.py` — Exports `AGREEMENT_LEVELS`, + `EnsembleAgreementResult`, `check_ensemble_agreement`, `run_ensemble_check_batch`. +- `src/openamp_foundry/cli/main.py` — Registered `simulation-ensemble-check` + subcommand with `--sequence`, `--results-json`, `--score-key`, `--threshold`, + `--format` flags. Added import and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_simulation_ensemble_check()` CLI handler with JSON parsing, + SimulationResult deserialization, text and JSON output. +- `Makefile` — Added `simulation-ensemble-check` target with demo invocation + using AKLWKR + two module results. Added to `.PHONY`. +- `tests/simulation/test_ensemble_checker.py` — 20 tests covering: empty + results, 1 result weak, 2 results moderate, 3+ results strong, conflict, + missing score_key skipped, mean_score, score_range, dry_lab_only always True, + scores_by_module, threshold parameter, modules_checked, agreement_description, + to_dict fields, batch counts, batch any_conflict, batch dry_lab_only. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.94 H6 changelog. Test count: 3201. +- `tests/test_test_count_regression.py` — baseline updated to 3201. + +Honest boundaries: +- Agreement is a computational measure: when N modules agree within threshold, + it means their scores are numerically close, not that they are biologically + correct. +- The threshold is a user-parameterized float; different thresholds produce + different agreement levels for the same data. +- A "strong" agreement from three weak modules is still weak evidence. The + checker measures consistency, not quality. +- Missing score_key entries are silently skipped — the checker does not validate + that the score_key is meaningful or calibrated. +- The dry_lab_only flag is a safety constraint, not a technical guarantee. +- Ensemble agreement does not constitute biological proof. All simulation + outputs remain dry-lab hypotheses requiring wet-lab validation. + ## v0.5.93 — Loop 93: Phase H H5 — Simulation-Result Provenance Chain ✓ (2026-07-09) `SimulationProvenanceRecord` dataclass with run_id, module_id, module_version, diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index c60f8201..b4f5a435 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1339,7 +1339,78 @@ def _run_simulation_registry(args: argparse.Namespace) -> int: print(f" Maintainer: {entry.maintainer}") if entry.notes: print(f" Notes: {entry.notes}") - return 0 + return 0 + + +def _run_simulation_ensemble_check(args: argparse.Namespace) -> int: + """Check agreement across multiple simulation results for a sequence.""" + import json + + from openamp_foundry.simulation.ensemble_checker import ( + check_ensemble_agreement, + ) + from openamp_foundry.simulation.interfaces import SimulationResult + + sequence = args.sequence + output_format = getattr(args, "format", "text") + score_key = args.score_key + threshold = args.threshold + + try: + raw_results = json.loads(args.results_json) + except (json.JSONDecodeError, TypeError) as e: + print(json.dumps({ + "status": "error", + "error": f"Invalid JSON for --results-json: {e}", + })) + return 2 + + if not isinstance(raw_results, list): + print(json.dumps({ + "status": "error", + "error": "--results-json must be a JSON array of SimulationResult dicts", + })) + return 2 + + results = [] + for item in raw_results: + results.append(SimulationResult( + module=item.get("module", ""), + version=item.get("version", ""), + scope=item.get("scope", []), + scores=item.get("scores", {}), + uncertainty=item.get("uncertainty", 0.0), + calibration_set=item.get("calibration_set"), + validated_against=item.get("validated_against", []), + notes=item.get("notes", []), + )) + + agreement = check_ensemble_agreement( + sequence=sequence, + results=results, + score_key=score_key, + threshold=threshold, + ) + + if output_format == "json": + print(json.dumps(agreement.to_dict(), indent=2)) + else: + print(f"Ensemble Agreement Check: {agreement.sequence}") + print(f" Modules checked: {', '.join(agreement.modules_checked)}") + print(f" Agreement level: {agreement.agreement_level}") + print(f" Description: {agreement.agreement_description}") + print(f" Mean score: {agreement.mean_score}") + print(f" Score range: {agreement.score_range}") + print(f" Threshold: {agreement.threshold}") + if agreement.scores_by_module: + print(f" Scores by module:") + for mod, sc in agreement.scores_by_module.items(): + print(f" {mod}: {sc}") + print() + print("Dry-lab only. Ensemble agreement is a computational measure,") + print("not biological proof.") + + return 0 if status_filter: if status_filter not in VALID_STATUSES: diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index f92a9b09..906d0246 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1490,6 +1490,35 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Simulation-ensemble agreement check (H6) ───────────────────────── + sec = sub.add_parser( + "simulation-ensemble-check", + help=( + "Check agreement across multiple simulation results for a sequence. " + "When multiple simulation modules independently agree on a candidate, " + "that agreement is stronger evidence than a single module alone. " + "Dry-lab only." + ), + ) + sec.add_argument("--sequence", type=str, required=True, help="Peptide sequence to check.") + sec.add_argument( + "--results-json", type=str, required=True, + help="JSON array of SimulationResult dicts.", + ) + sec.add_argument( + "--score-key", type=str, default="binding_energy", + help="Score key to extract from each result's scores dict (default: binding_energy).", + ) + sec.add_argument( + "--threshold", type=float, default=0.2, + help="Agreement threshold (max score range for agreement, default: 0.2).", + ) + sec.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + return parser @@ -1660,6 +1689,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "simulation-provenance": return _run_simulation_provenance(args) + if args.command == "simulation-ensemble-check": + return _run_simulation_ensemble_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/simulation/__init__.py b/src/openamp_foundry/simulation/__init__.py index 7d59d5d9..96662a70 100644 --- a/src/openamp_foundry/simulation/__init__.py +++ b/src/openamp_foundry/simulation/__init__.py @@ -6,6 +6,12 @@ evaluate_adapter_gate, run_adapter_gate_batch, ) +from .ensemble_checker import ( + AGREEMENT_LEVELS, + EnsembleAgreementResult, + check_ensemble_agreement, + run_ensemble_check_batch, +) from .provenance import ( SimulationProvenanceRecord, make_provenance_record, @@ -48,6 +54,10 @@ "FAIL_CLOSED_REASONS", "evaluate_adapter_gate", "run_adapter_gate_batch", + "AGREEMENT_LEVELS", + "EnsembleAgreementResult", + "check_ensemble_agreement", + "run_ensemble_check_batch", "SimulationResult", "VirtualAssayProxy", "ExternalSimulationAdapter", diff --git a/src/openamp_foundry/simulation/ensemble_checker.py b/src/openamp_foundry/simulation/ensemble_checker.py new file mode 100644 index 00000000..903e3565 --- /dev/null +++ b/src/openamp_foundry/simulation/ensemble_checker.py @@ -0,0 +1,164 @@ +"""Ensemble agreement checker for simulation modules. + +When multiple simulation modules independently agree on a candidate, +that agreement is stronger evidence than a single module alone. +This module makes agreement explicit and auditable. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, asdict +from statistics import mean +from typing import Any + +from .interfaces import SimulationResult + + +AGREEMENT_LEVELS: dict[str, str] = { + "strong": "≥3 modules agree within threshold", + "moderate": "2 modules agree within threshold", + "weak": "Only 1 module; no ensemble signal", + "conflict": "Modules disagree beyond threshold", + "insufficient": "No results provided", +} + + +@dataclass +class EnsembleAgreementResult: + sequence: str + modules_checked: list[str] + agreement_level: str + agreement_description: str + mean_score: float | None + score_range: float | None + scores_by_module: dict[str, float] + threshold: float + dry_lab_only: bool = True + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def check_ensemble_agreement( + sequence: str, + results: list[SimulationResult], + score_key: str = "binding_energy", + threshold: float = 0.2, +) -> EnsembleAgreementResult: + """Check agreement across multiple simulation results. + + Logic: + - If results is empty -> agreement_level="insufficient" + - Extract score_key from each result's scores dict (skip if missing) + - If only 1 score found -> agreement_level="weak" + - Compute score_range = max(scores) - min(scores) + - If score_range <= threshold: + n = len(scores) + agreement_level = "strong" if n >= 3 else "moderate" + - Else: agreement_level = "conflict" + - mean_score = mean of available scores (None if 0 scores) + - score_range = None if 0 scores + - scores_by_module: dict mapping result.module -> score for results with score_key + """ + modules_checked = [r.module for r in results] + scores_by_module: dict[str, float] = {} + + for result in results: + if score_key in result.scores: + scores_by_module[result.module] = result.scores[score_key] + + scores = list(scores_by_module.values()) + n = len(scores) + + if n == 0: + return EnsembleAgreementResult( + sequence=sequence, + modules_checked=modules_checked, + agreement_level="insufficient", + agreement_description=AGREEMENT_LEVELS["insufficient"], + mean_score=None, + score_range=None, + scores_by_module=scores_by_module, + threshold=threshold, + ) + + if n == 1: + score = scores[0] + return EnsembleAgreementResult( + sequence=sequence, + modules_checked=modules_checked, + agreement_level="weak", + agreement_description=AGREEMENT_LEVELS["weak"], + mean_score=score, + score_range=0.0, + scores_by_module=scores_by_module, + threshold=threshold, + ) + + score_range_val = max(scores) - min(scores) + mean_score_val = mean(scores) + + if score_range_val <= threshold: + agreement_key = "strong" if n >= 3 else "moderate" + else: + agreement_key = "conflict" + + return EnsembleAgreementResult( + sequence=sequence, + modules_checked=modules_checked, + agreement_level=agreement_key, + agreement_description=AGREEMENT_LEVELS[agreement_key], + mean_score=mean_score_val, + score_range=score_range_val, + scores_by_module=scores_by_module, + threshold=threshold, + ) + + +def run_ensemble_check_batch( + calls: list[dict], +) -> dict: + """Run ensemble agreement checks for a batch of calls. + + Each call dict has: + - "sequence": str (required) + - "results": list[SimulationResult] (required) + - "score_key": str (optional, default "binding_energy") + - "threshold": float (optional, default 0.2) + + Returns dict with: + - total, strong, moderate, weak, conflict, insufficient counts + - any_conflict: True if any call returns "conflict" + - results: list[EnsembleAgreementResult as dict] + - dry_lab_only: True + """ + results_list: list[EnsembleAgreementResult] = [] + counts: dict[str, int] = { + "strong": 0, + "moderate": 0, + "weak": 0, + "conflict": 0, + "insufficient": 0, + } + + for call in calls: + result = check_ensemble_agreement( + sequence=call["sequence"], + results=call["results"], + score_key=call.get("score_key", "binding_energy"), + threshold=call.get("threshold", 0.2), + ) + results_list.append(result) + counts[result.agreement_level] += 1 + + return { + "total": len(calls), + "strong": counts["strong"], + "moderate": counts["moderate"], + "weak": counts["weak"], + "conflict": counts["conflict"], + "insufficient": counts["insufficient"], + "any_conflict": counts["conflict"] > 0, + "results": [r.to_dict() for r in results_list], + "dry_lab_only": True, + } diff --git a/tests/simulation/test_ensemble_checker.py b/tests/simulation/test_ensemble_checker.py new file mode 100644 index 00000000..3717d285 --- /dev/null +++ b/tests/simulation/test_ensemble_checker.py @@ -0,0 +1,235 @@ +"""Tests for simulation ensemble agreement checker.""" +from __future__ import annotations + +import pytest + +from openamp_foundry.simulation.ensemble_checker import ( + AGREEMENT_LEVELS, + EnsembleAgreementResult, + check_ensemble_agreement, + run_ensemble_check_batch, +) +from openamp_foundry.simulation.interfaces import SimulationResult + + +def _make_result( + module: str, + score: float, + score_key: str = "binding_energy", +) -> SimulationResult: + return SimulationResult( + module=module, + version="0.1.0", + scope=["bacterial_binding"], + scores={score_key: score}, + uncertainty=0.1, + calibration_set=None, + validated_against=["APD3"], + notes=[], + ) + + +class TestAgreementLevels: + def test_sufficient_keys_present(self): + expected_keys = {"strong", "moderate", "weak", "conflict", "insufficient"} + assert set(AGREEMENT_LEVELS.keys()) == expected_keys + + def test_descriptions_non_empty(self): + for desc in AGREEMENT_LEVELS.values(): + assert desc and isinstance(desc, str) + + +class TestCheckEnsembleAgreement: + def test_empty_results_insufficient(self): + result = check_ensemble_agreement("AKLWKR", [], score_key="binding_energy") + assert result.agreement_level == "insufficient" + assert result.mean_score is None + assert result.score_range is None + assert result.scores_by_module == {} + + def test_one_result_weak(self): + results = [_make_result("membrane_proxy", 0.75)] + result = check_ensemble_agreement("AKLWKR", results) + assert result.agreement_level == "weak" + assert result.mean_score == 0.75 + assert result.score_range == 0.0 + + def test_two_results_within_threshold_moderate(self): + results = [ + _make_result("membrane_proxy", 0.75), + _make_result("structure_proxy", 0.80), + ] + result = check_ensemble_agreement("AKLWKR", results, threshold=0.2) + assert result.agreement_level == "moderate" + assert result.score_range <= 0.2 + + def test_three_results_within_threshold_strong(self): + results = [ + _make_result("membrane_proxy", 0.75), + _make_result("structure_proxy", 0.80), + _make_result("docking_proxy", 0.78), + ] + result = check_ensemble_agreement("AKLWKR", results, threshold=0.2) + assert result.agreement_level == "strong" + assert result.score_range <= 0.2 + + def test_results_outside_threshold_conflict(self): + results = [ + _make_result("membrane_proxy", 0.75), + _make_result("structure_proxy", 0.99), + ] + result = check_ensemble_agreement("AKLWKR", results, threshold=0.2) + assert result.agreement_level == "conflict" + assert result.score_range > 0.2 + + def test_missing_score_key_skipped_gracefully(self): + results = [ + _make_result("membrane_proxy", 0.75), + SimulationResult( + module="structure_proxy", + version="0.1.0", + scope=["bacterial_binding"], + scores={"other_key": 0.80}, + uncertainty=0.1, + calibration_set=None, + validated_against=["APD3"], + notes=[], + ), + ] + result = check_ensemble_agreement("AKLWKR", results, score_key="binding_energy") + assert result.agreement_level == "weak" + assert result.mean_score == 0.75 + assert "membrane_proxy" in result.scores_by_module + assert "structure_proxy" not in result.scores_by_module + + def test_mean_score_correct_for_two_results(self): + results = [ + _make_result("membrane_proxy", 0.70), + _make_result("structure_proxy", 0.90), + ] + result = check_ensemble_agreement("AKLWKR", results, threshold=0.2) + assert result.mean_score == 0.80 + + def test_score_range_correct(self): + results = [ + _make_result("membrane_proxy", 0.70), + _make_result("structure_proxy", 0.90), + ] + result = check_ensemble_agreement("AKLWKR", results, threshold=0.2) + assert result.score_range == pytest.approx(0.20) + + def test_dry_lab_only_always_true(self): + results = [ + _make_result("membrane_proxy", 0.75), + _make_result("structure_proxy", 0.80), + ] + result = check_ensemble_agreement("AKLWKR", results) + assert result.dry_lab_only is True + + def test_scores_by_module_populated(self): + results = [ + _make_result("membrane_proxy", 0.75), + _make_result("structure_proxy", 0.80), + ] + result = check_ensemble_agreement("AKLWKR", results) + assert result.scores_by_module == { + "membrane_proxy": 0.75, + "structure_proxy": 0.80, + } + + def test_threshold_parameter_respected(self): + results = [ + _make_result("membrane_proxy", 0.70), + _make_result("structure_proxy", 0.85), + ] + wide = check_ensemble_agreement("AKLWKR", results, threshold=0.2) + narrow = check_ensemble_agreement("AKLWKR", results, threshold=0.1) + assert wide.agreement_level == "moderate" + assert narrow.agreement_level == "conflict" + + def test_modules_checked_includes_all_modules(self): + results = [ + _make_result("membrane_proxy", 0.75), + _make_result("structure_proxy", 0.80), + ] + result = check_ensemble_agreement("AKLWKR", results) + assert set(result.modules_checked) == {"membrane_proxy", "structure_proxy"} + + def test_agreement_description_matches_level(self): + results = [ + _make_result("membrane_proxy", 0.75), + _make_result("structure_proxy", 0.80), + ] + result = check_ensemble_agreement("AKLWKR", results, threshold=0.2) + assert result.agreement_description == AGREEMENT_LEVELS[result.agreement_level] + + def test_to_dict_contains_all_fields(self): + results = [ + _make_result("membrane_proxy", 0.75), + _make_result("structure_proxy", 0.80), + ] + result = check_ensemble_agreement("AKLWKR", results) + d = result.to_dict() + assert d["sequence"] == "AKLWKR" + assert d["agreement_level"] == "moderate" + assert d["dry_lab_only"] is True + + +class TestRunEnsembleCheckBatch: + def test_batch_counts_correct(self): + calls = [ + {"sequence": "SEQ1", "results": [], "score_key": "binding_energy"}, + { + "sequence": "SEQ2", + "results": [_make_result("membrane_proxy", 0.75)], + "score_key": "binding_energy", + }, + { + "sequence": "SEQ3", + "results": [ + _make_result("membrane_proxy", 0.75), + _make_result("structure_proxy", 0.80), + ], + "score_key": "binding_energy", + "threshold": 0.2, + }, + ] + batch = run_ensemble_check_batch(calls) + assert batch["total"] == 3 + assert batch["insufficient"] == 1 + assert batch["weak"] == 1 + assert batch["moderate"] == 1 + + def test_batch_any_conflict_true_when_conflict(self): + calls = [ + { + "sequence": "SEQ1", + "results": [ + _make_result("membrane_proxy", 0.70), + _make_result("structure_proxy", 0.99), + ], + "score_key": "binding_energy", + "threshold": 0.2, + }, + ] + batch = run_ensemble_check_batch(calls) + assert batch["conflict"] == 1 + assert batch["any_conflict"] is True + + def test_batch_dry_lab_only_true(self): + calls = [ + {"sequence": "SEQ1", "results": []}, + ] + batch = run_ensemble_check_batch(calls) + assert batch["dry_lab_only"] is True + + def test_batch_results_list_length_matches_total(self): + calls = [ + {"sequence": "SEQ1", "results": []}, + { + "sequence": "SEQ2", + "results": [_make_result("membrane_proxy", 0.75)], + }, + ] + batch = run_ensemble_check_batch(calls) + assert len(batch["results"]) == 2 diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 64c951c6..b120abf2 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3181 tests as of Loop 93 (2026-07-09) -- Phase H H5 (simulation-result provenance chain) +# Baseline: 3201 tests as of Loop 94 (2026-07-09) -- Phase H H6 (simulation-ensemble agreement checker) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3181 +BASELINE = 3201 TOLERANCE = 0.05 # allow 5% fluctuation