feat: Phase H H6 — simulation-ensemble agreement checker

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance
+.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check
 
 PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
 PYTEST  := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
@@ -745,3 +745,9 @@ simulation-provenance:
 		--input-sequence AKLWKR \
 		--scores-json '{"binding_energy": 0.75}'
 	@echo "Provenance record complete."
+
+simulation-ensemble-check:
+	.venv/bin/openamp-foundry simulation-ensemble-check \
+		--sequence AKLWKR \
+		--results-json '[{"module":"membrane_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.75},"uncertainty":0.1,"calibration_set":null,"validated_against":["APD3"],"notes":[]},{"module":"structure_proxy","version":"0.1.0","scope":["bacterial_binding"],"scores":{"binding_energy":0.80},"uncertainty":0.1,"calibration_set":null,"validated_against":["APD3"],"notes":[]}]'
+	@echo "Ensemble check complete."

docs/evidence/METRICS_CURRENT.md

@@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
 > with this file, this file wins. Updated whenever benchmark/benchmark config changes.
 >
-> **Last updated:** 2026-07-09 (Phase H H5 — simulation-result provenance chain — v0.5.93)
+> **Last updated:** 2026-07-09 (Phase H H6 — simulation-ensemble agreement checker — v0.5.94)
+> **New in v0.5.94:** Simulation-ensemble agreement checker (H6) — `ensemble_checker.py` with `EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked, agreement_level, agreement_description, mean_score, score_range, scores_by_module, threshold, dry_lab_only). `AGREEMENT_LEVELS` dict (5 levels: strong, moderate, weak, conflict, insufficient). `check_ensemble_agreement()` extracts score_key from each SimulationResult's scores dict, computes score_range, classifies into strong (≥3 modules within threshold), moderate (2 modules within threshold), weak (1 module), conflict (beyond threshold), or insufficient (no results). `run_ensemble_check_batch()` aggregates multiple calls with counts and any_conflict flag. CLI (`openamp-foundry simulation-ensemble-check`) with `--sequence`, `--results-json`, `--score-key`, `--threshold`, `--format text|json`. `make simulation-ensemble-check` target. 20 tests. **3201 total.** When multiple simulation modules independently agree on a candidate, that agreement is stronger evidence than a single module alone. The ensemble checker makes this agreement explicit and auditable.
 > **New in v0.5.93:** Simulation-result provenance chain (H5) — `SimulationProvenanceRecord` dataclass with run_id, module_id, module_version, timestamp_utc, input_hash (SHA-256 of input sequence), result_hash (SHA-256 of sorted scores dict), calibration_set, notes, and dry_lab_only. `make_provenance_record()` computes input_hash and result_hash deterministically (sort_keys=True for result_hash). `validate_provenance_record()` checks run_id, module_id, module_version non-empty; ISO 8601 timestamp contains 'T'; hashes are 64-char hex; dry_lab_only is True. `provenance_summary()` aggregates total, unique modules, run_ids, and dry_lab_only flag. CLI (`openamp-foundry simulation-provenance`) with `--run-id`, `--module-id`, `--module-version`, `--timestamp-utc`, `--input-sequence`, `--scores-json`, `--calibration-set`, `--format text|json`. `make simulation-provenance` target with demo invocation. 19 tests. **3181 total.** Every simulation result carries a traceable provenance chain so results can be audited, reproduced, or invalidated later without relying on memory.
 > **New in v0.5.92:** Fail-closed adapter integration tests (H4) — `FAIL_CLOSED_REASONS` dict keys (6 entries: timeout, connection_refused, invalid_response, schema_violation, module_unavailable, baseline_not_beaten). `AdapterGateResult` dataclass with module_id, passed, failure_reason, failure_detail, dry_lab_only. `evaluate_adapter_gate()` fail-closed: returns passed=False on ANY failure signal with priority ordering (timeout > connection_refused > invalid_response > schema_violation > module_unavailable > baseline_not_beaten). `run_adapter_gate_batch()` aggregates multiple adapter calls with total/passed/failed/any_failed/results/dry_lab_only. CLI (`openamp-foundry adapter-gate-check`) with `--module-id`, `--timeout`, `--connection-refused`, `--schema-errors`, `--module-unavailable`, `--baseline-beaten`, `--format text|json`. `make adapter-gate-check` target. 21 tests. **3162 total.** Avoids hidden external failures — when the adapter to an external simulation service is down or misbehaves, the pipeline must fail loudly rather than silently passing garbage through.
 > **New in v0.5.91:** Per-module cheapest-baseline declaration (H3) — `BaselineDeclaration` dataclass with module_id, module_name, baseline_description, baseline_type, evidence_level_ceiling, and notes. `BASELINE_DECLARATIONS` list (4 entries: membrane_proxy, structure_proxy, dummy_membrane_proxy, external_adapter_placeholder). `get_baseline_declaration()` and `list_baseline_declarations()` for lookup. `check_baseline_requirement()` caps effective_evidence_level to ceiling when baseline not beaten; returns dict with module_id, baseline_beaten, claimed/effective evidence levels, capped flag, message, and dry_lab_only. `validate_baseline_declarations()` checks module_id non-empty, baseline_description non-empty, baseline_type in {heuristic, random, constant, length}, evidence_level_ceiling 1-6, no duplicate module_ids. CLI (`openamp-foundry simulation-baseline-check`) with `--module-id`, `--claimed-level`, `--baseline-beaten`, `--format text|json`. `make simulation-baseline-check` target. 16 tests. **3141 total.** Forces honest enemy comparison — every simulation module must declare the simplest baseline it must beat.
@@ -84,7 +85,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test).
 > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination.
 > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison.
-> **Pipeline version:** v0.5.93 (H5 — simulation-result provenance chain)
+> **Pipeline version:** v0.5.94 (H6 — simulation-ensemble agreement checker)
 > **Branch:** main
 
 ---
@@ -1154,7 +1155,7 @@ Decoys score low on activity. Selective AMPs score moderately on both.
 
 | Metric | Value |
 |--------|-------|
-| Total tests | 3162 |
+| Total tests | 3201 |
 | Coverage (branch) | 99% (6 CLI guard lines only) |
 | Source modules at 100% | All pipeline, QC, scoring modules |
 

docs/research/ROADMAP.md

@@ -1,5 +1,59 @@
 # Roadmap
 
+## v0.5.94 — Loop 94: Phase H H6 — Simulation-Ensemble Agreement Checker ✓ (2026-07-09)
+
+`EnsembleAgreementResult` dataclass (9 fields: sequence, modules_checked,
+agreement_level, agreement_description, mean_score, score_range, scores_by_module,
+threshold, dry_lab_only). `AGREEMENT_LEVELS` dict (5 levels: strong/moderate/
+weak/conflict/insufficient). `check_ensemble_agreement()` extracts score_key from
+each SimulationResult's scores dict, computes score_range, classifies agreement:
+strong (≥3 modules within threshold), moderate (2 modules within threshold),
+weak (1 module), conflict (beyond threshold), or insufficient (no results).
+`run_ensemble_check_batch()` aggregates multiple calls with counts and any_conflict
+flag. CLI (`openamp-foundry simulation-ensemble-check`) with `--sequence`,
+`--results-json`, `--score-key`, `--threshold`, `--format text|json`.
+`make simulation-ensemble-check` target with demo invocation. When multiple
+simulation modules independently agree on a candidate, that agreement is stronger
+evidence than a single module alone. The ensemble checker makes this agreement
+explicit and auditable.
+
+Changes:
+- `src/openamp_foundry/simulation/ensemble_checker.py` (H6) — Core module with
+  `AGREEMENT_LEVELS` dict (5 entries), `EnsembleAgreementResult` dataclass
+  (9 fields), `check_ensemble_agreement()` with 5-path classification logic,
+  `run_ensemble_check_batch()` with per-level counts and any_conflict flag.
+- `src/openamp_foundry/simulation/__init__.py` — Exports `AGREEMENT_LEVELS`,
+  `EnsembleAgreementResult`, `check_ensemble_agreement`, `run_ensemble_check_batch`.
+- `src/openamp_foundry/cli/main.py` — Registered `simulation-ensemble-check`
+  subcommand with `--sequence`, `--results-json`, `--score-key`, `--threshold`,
+  `--format` flags. Added import and dispatch.
+- `src/openamp_foundry/cli/commands/reports.py` — Added
+  `_run_simulation_ensemble_check()` CLI handler with JSON parsing,
+  SimulationResult deserialization, text and JSON output.
+- `Makefile` — Added `simulation-ensemble-check` target with demo invocation
+  using AKLWKR + two module results. Added to `.PHONY`.
+- `tests/simulation/test_ensemble_checker.py` — 20 tests covering: empty
+  results, 1 result weak, 2 results moderate, 3+ results strong, conflict,
+  missing score_key skipped, mean_score, score_range, dry_lab_only always True,
+  scores_by_module, threshold parameter, modules_checked, agreement_description,
+  to_dict fields, batch counts, batch any_conflict, batch dry_lab_only.
+- `docs/evidence/METRICS_CURRENT.md` — v0.5.94 H6 changelog. Test count: 3201.
+- `tests/test_test_count_regression.py` — baseline updated to 3201.
+
+Honest boundaries:
+- Agreement is a computational measure: when N modules agree within threshold,
+  it means their scores are numerically close, not that they are biologically
+  correct.
+- The threshold is a user-parameterized float; different thresholds produce
+  different agreement levels for the same data.
+- A "strong" agreement from three weak modules is still weak evidence. The
+  checker measures consistency, not quality.
+- Missing score_key entries are silently skipped — the checker does not validate
+  that the score_key is meaningful or calibrated.
+- The dry_lab_only flag is a safety constraint, not a technical guarantee.
+- Ensemble agreement does not constitute biological proof. All simulation
+  outputs remain dry-lab hypotheses requiring wet-lab validation.
+
 ## v0.5.93 — Loop 93: Phase H H5 — Simulation-Result Provenance Chain ✓ (2026-07-09)
 
 `SimulationProvenanceRecord` dataclass with run_id, module_id, module_version,

src/openamp_foundry/cli/commands/reports.py

@@ -1339,7 +1339,78 @@ def _run_simulation_registry(args: argparse.Namespace) -> int:
             print(f"  Maintainer:         {entry.maintainer}")
             if entry.notes:
                 print(f"  Notes:              {entry.notes}")
-        return 0
+    return 0
+
+
+def _run_simulation_ensemble_check(args: argparse.Namespace) -> int:
+    """Check agreement across multiple simulation results for a sequence."""
+    import json
+
+    from openamp_foundry.simulation.ensemble_checker import (
+        check_ensemble_agreement,
+    )
+    from openamp_foundry.simulation.interfaces import SimulationResult
+
+    sequence = args.sequence
+    output_format = getattr(args, "format", "text")
+    score_key = args.score_key
+    threshold = args.threshold
+
+    try:
+        raw_results = json.loads(args.results_json)
+    except (json.JSONDecodeError, TypeError) as e:
+        print(json.dumps({
+            "status": "error",
+            "error": f"Invalid JSON for --results-json: {e}",
+        }))
+        return 2
+
+    if not isinstance(raw_results, list):
+        print(json.dumps({
+            "status": "error",
+            "error": "--results-json must be a JSON array of SimulationResult dicts",
+        }))
+        return 2
+
+    results = []
+    for item in raw_results:
+        results.append(SimulationResult(
+            module=item.get("module", ""),
+            version=item.get("version", ""),
+            scope=item.get("scope", []),
+            scores=item.get("scores", {}),
+            uncertainty=item.get("uncertainty", 0.0),
+            calibration_set=item.get("calibration_set"),
+            validated_against=item.get("validated_against", []),
+            notes=item.get("notes", []),
+        ))
+
+    agreement = check_ensemble_agreement(
+        sequence=sequence,
+        results=results,
+        score_key=score_key,
+        threshold=threshold,
+    )
+
+    if output_format == "json":
+        print(json.dumps(agreement.to_dict(), indent=2))
+    else:
+        print(f"Ensemble Agreement Check: {agreement.sequence}")
+        print(f"  Modules checked:  {', '.join(agreement.modules_checked)}")
+        print(f"  Agreement level:  {agreement.agreement_level}")
+        print(f"  Description:      {agreement.agreement_description}")
+        print(f"  Mean score:       {agreement.mean_score}")
+        print(f"  Score range:      {agreement.score_range}")
+        print(f"  Threshold:        {agreement.threshold}")
+        if agreement.scores_by_module:
+            print(f"  Scores by module:")
+            for mod, sc in agreement.scores_by_module.items():
+                print(f"    {mod}: {sc}")
+        print()
+        print("Dry-lab only. Ensemble agreement is a computational measure,")
+        print("not biological proof.")
+
+    return 0
 
     if status_filter:
         if status_filter not in VALID_STATUSES:

src/openamp_foundry/cli/main.py

@@ -5,7 +5,7 @@
 from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale
 from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus
 from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc
-from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance
+from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check
 from openamp_foundry.cli.commands.gates import _run_gate_check
 
 import argparse
@@ -1490,6 +1490,35 @@ def build_parser() -> argparse.ArgumentParser:
         help="Output format (default: text).",
     )
 
+    # ── Simulation-ensemble agreement check (H6) ─────────────────────────
+    sec = sub.add_parser(
+        "simulation-ensemble-check",
+        help=(
+            "Check agreement across multiple simulation results for a sequence. "
+            "When multiple simulation modules independently agree on a candidate, "
+            "that agreement is stronger evidence than a single module alone. "
+            "Dry-lab only."
+        ),
+    )
+    sec.add_argument("--sequence", type=str, required=True, help="Peptide sequence to check.")
+    sec.add_argument(
+        "--results-json", type=str, required=True,
+        help="JSON array of SimulationResult dicts.",
+    )
+    sec.add_argument(
+        "--score-key", type=str, default="binding_energy",
+        help="Score key to extract from each result's scores dict (default: binding_energy).",
+    )
+    sec.add_argument(
+        "--threshold", type=float, default=0.2,
+        help="Agreement threshold (max score range for agreement, default: 0.2).",
+    )
+    sec.add_argument(
+        "--format", type=str, default="text",
+        choices=["text", "json"],
+        help="Output format (default: text).",
+    )
+
     return parser
 
 
@@ -1660,6 +1689,9 @@ def main(argv: list[str] | None = None) -> int:
     if args.command == "simulation-provenance":
         return _run_simulation_provenance(args)
 
+    if args.command == "simulation-ensemble-check":
+        return _run_simulation_ensemble_check(args)
+
     parser.error("unknown command")
     return 2