cschanhniem · Jul 9, 2026 · Jul 9, 2026
diff --git a/Makefile b/Makefile
@@ -963,6 +963,9 @@ negative-result-check:
 experiment-priority-check:
 	openamp-foundry experiment-priority-check --entry-json '{"justification_id":"EPJ-001","batch_id":"BATCH-002","pipeline_version":"0.9.8","decision_date":"2026-07-10","selection_criteria":["Highest predicted hit rate","All candidates passed safety filters"],"rejected_alternatives":["BATCH-001"],"rejection_rationale":"BATCH-001 showed lower predicted hit rate and higher predicted toxicity scores.","resource_constraint":"Synthesis limited to 10 candidates per wave.","safety_reviewed":true,"pre_specified":true,"decided_by":"pipeline@example.com","dry_lab_only":true}' --format text
 
+calibration-performance-check:
+	openamp-foundry calibration-performance-check --entry-json '{"summary_id":"CPS-001","pipeline_version":"0.9.9","evaluation_date":"2026-07-10","batch_ids_evaluated":["BATCH-001","BATCH-002"],"total_candidates_evaluated":20,"true_positive_count":10,"false_positive_count":2,"true_negative_count":6,"false_negative_count":2,"brier_score":0.12,"calibration_notes":"Calibration looks reasonable.","reviewer":"test@example.com","dry_lab_only":false}' --format text
+
 dataset-release-check:
 	.venv/bin/openamp-foundry dataset-release-check \
 		--entry-json '{"release_id":"DSR-001","dataset_name":"OpenAMP Wave 0.5 Candidates","dataset_version":"1.0.0","release_date":"2026-07-09","license_identifier":"CC-BY-4.0","data_sources":["UniProt AMP database","APD3","in-house pipeline"],"contains_sequences":true,"contains_activity_data":false,"dual_use_assessed":true,"usage_policy_url":"https://openamp.example.org/data-policy","contact_email":"data@openamp.example.org","release_approved":true,"dry_lab_only":true}' \

diff --git a/docs/evidence/CALIBRATION_PERFORMANCE_GUIDE.md b/docs/evidence/CALIBRATION_PERFORMANCE_GUIDE.md
@@ -0,0 +1,62 @@
+# Calibration Performance Summary Guide
+
+## Purpose
+
+A calibration performance summary aggregates prediction accuracy metrics
+across a window of experimental batches where outcomes are known. It answers:
+"How well is the pipeline predicting real experimental results?"
+
+Without this record, calibration problems are invisible. A pipeline can be
+confidently wrong for many batches before anyone notices.
+
+## Schema Fields
+
+| Field | Type | Description |
+|---|---|---|
+| summary_id | str | Unique identifier (prefix "CPS-") |
+| pipeline_version | str | Pipeline version being evaluated |
+| evaluation_date | str | ISO 8601 date of this evaluation |
+| batch_ids_evaluated | list[str] | Batches included in this summary (at least 1) |
+| total_candidates_evaluated | int | Total candidates with known outcomes |
+| true_positive_count | int | Predicted active, confirmed active |
+| false_positive_count | int | Predicted active, confirmed inactive |
+| true_negative_count | int | Predicted inactive, confirmed inactive |
+| false_negative_count | int | Predicted inactive, confirmed active |
+| brier_score | float | Probability calibration score (0.0=perfect, 1.0=worst) |
+| calibration_notes | str | Observations about calibration quality (max 400 chars) |
+| reviewer | str | Person or system that validated this summary |
+| dry_lab_only | bool | False (uses real lab outcome data) |
+
+## Derived Metrics (computed, not stored)
+
+From the confusion matrix counts:
+- Precision = TP / (TP + FP)
+- Recall = TP / (TP + FN)
+- Specificity = TN / (TN + FP)
+- F1 = 2 * Precision * Recall / (Precision + Recall)
+
+## Validation Rules
+
+1. summary_id must start with "CPS-"
+2. batch_ids_evaluated must contain at least 1 batch
+3. total_candidates_evaluated must be at least 1
+4. true_positive_count, false_positive_count, true_negative_count, false_negative_count must all be >= 0
+5. total_candidates_evaluated must equal TP + FP + TN + FN
+6. brier_score must be between 0.0 and 1.0 (inclusive)
+7. calibration_notes must not exceed 400 characters
+8. reviewer must not be empty
+9. dry_lab_only must be False (requires real lab data)
+
+## Warning Conditions
+
+- High false positive rate: FP / (FP + TN) > 0.5 → warn that pipeline is over-predicting activity
+- Low recall: TP / (TP + FN) < 0.3 → warn that pipeline is missing many active candidates
+- Poor Brier score: brier_score > 0.25 → warn that probability calibration is poor
+- Small evaluation window: total_candidates_evaluated < 10 → warn that sample is too small for reliable calibration assessment
+
+## Honest Use Boundary
+
+Calibration performance is evaluated on *known* outcomes — candidates where
+experimental results have been received. Do not extrapolate calibration
+statistics to candidates not yet tested. A good Brier score on past batches
+does not guarantee accurate predictions for future batches.
diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md
@@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
 > with this file, this file wins. Updated whenever benchmark/benchmark config changes.
 >
-> **Last updated:** 2026-07-10 (Phase N N5 — experiment priority justification — v0.9.8)
+> **Last updated:** 2026-07-10 (Phase O O1 — calibration performance summary — v0.9.9)
+> **New in v0.9.9:** Calibration performance summary schema (O1 — Loop 138) — docs/evidence/CALIBRATION_PERFORMANCE_GUIDE.md with purpose, schema fields table (13 fields: summary_id, pipeline_version, evaluation_date, batch_ids_evaluated, total_candidates_evaluated, true_positive_count, false_positive_count, true_negative_count, false_negative_count, brier_score, calibration_notes, reviewer, dry_lab_only), 4 derived metrics (precision, recall, specificity, F1), 9 validation rules (CPS- prefix, >=1 batch, >=1 candidates, all counts >=0, total == TP+FP+TN+FN, Brier 0.0-1.0, notes<=400 chars, non-empty reviewer, dry_lab_only must be False), 4 warning conditions (high FP rate >0.5, low recall <0.3, poor Brier >0.25, small sample <10), honest-use boundary. CalibrationPerformanceEntry dataclass (13 fields, dry_lab_only=False), CalibrationPerformanceResult dataclass (8 fields: summary_id, pipeline_version, total_candidates_evaluated, brier_score, passed, errors, warnings, dry_lab_only=False), MAX_CALIBRATION_NOTES_LENGTH=400, MIN_CANDIDATES_FOR_RELIABLE_ESTIMATE=10, HIGH_FP_RATE_THRESHOLD=0.5, LOW_RECALL_THRESHOLD=0.3, POOR_BRIER_SCORE_THRESHOLD=0.25. validate_calibration_performance() (9+ error checks: CPS- prefix, >=1 batch, >=1 candidates, >=0 counts, confusion sum match, Brier bounds, notes<=400, non-empty reviewer, dry_lab_only must be False; 4 warnings: high FP, low recall, poor Brier, small sample). validate_calibration_performance_dict() (12 required fields guard). CLI (openamp-foundry calibration-performance-check) with --entry-json, --format text|json. make calibration-performance-check target. 68 tests. Phase O O1: calibration performance summaries now measure how well predictions match experimental outcomes. Phase O (Calibration Quality Assurance) started.
 > **New in v0.9.8:** Experiment priority justification schema (N5 — Loop 137) — docs/evidence/EXPERIMENT_PRIORITY_GUIDE.md with purpose, schema fields table (12 fields: justification_id, batch_id, pipeline_version, decision_date, selection_criteria, rejected_alternatives, rejection_rationale, resource_constraint, safety_reviewed, pre_specified, decided_by, dry_lab_only), 8 validation rules (EPJ- prefix, >=2 selection criteria, >=1 rejected alternative, non-empty rejection_rationale<=500 chars, resource_constraint<=200 chars, safety_reviewed must be True, non-empty decided_by, dry_lab_only must be True), 4 warning conditions (post-hoc, minimum criteria, no resource constraint, many criteria), honest-use boundary. ExperimentPriorityEntry dataclass (12 fields, dry_lab_only=True), ExperimentPriorityResult dataclass (8 fields: justification_id, batch_id, criteria_count, rejected_alternative_count, safety_reviewed, passed, errors, warnings, dry_lab_only=True), MINIMUM_SELECTION_CRITERIA=2, MAXIMUM_SELECTION_CRITERIA_WARNING=6, MAX_REJECTION_RATIONALE_LENGTH=500, MAX_RESOURCE_CONSTRAINT_LENGTH=200. validate_experiment_priority() (8+ error checks: EPJ- prefix, >=2 criteria, >=1 alternative, non-empty rationale<=500 chars, constraint<=200 chars, safety_reviewed must be True, non-empty decided_by, dry_lab_only must be True; 4 warnings: post-hoc, minimum criteria, no constraint, many criteria). validate_experiment_priority_dict() (11 required fields guard). CLI (openamp-foundry experiment-priority-check) with --entry-json, --format text|json. make experiment-priority-check target. 62 tests. Phase N N5: batch selection decisions are now documented with full audit trail over alternatives. Phase N (Pre-registration & Baseline Honesty) is fully complete with N1-N5.
 > **New in v0.9.7:** Negative result record schema (N4 — Loop 136) — docs/evidence/NEGATIVE_RESULT_GUIDE.md with purpose, schema fields table (14 fields: record_id, batch_id, pipeline_version, record_date, failure_category, failure_description, candidate_ids, assay_type, expected_outcome, observed_outcome, hypothesis_impact, will_be_reported, recorded_by, dry_lab_only), 6 valid failure categories (assay_quality_failure, below_activity_threshold, excessive_toxicity, model_overprediction, pipeline_error, stability_failure), 5 valid assay types (cytotoxicity_assay, hemolysis_assay, membrane_disruption_assay, mic_assay, stability_assay), 9 validation rules, 3 warning conditions (not reported, large failure set, model overprediction without recalibration), honest-use boundary. NegativeResultEntry dataclass (14 fields), NegativeResultResult dataclass (7 fields), validate_negative_result() (9+ error checks: NRR- prefix, valid failure_category, non-empty failure_description<=500 chars, >=1 candidate_ids, valid assay_type, non-empty expected_outcome/observed_outcome, non-empty hypothesis_impact<=300 chars, non-empty recorded_by; 3 warnings: not reported contributes to publication bias, >10 candidates suggests systematic issue, model_overprediction without calibration mention triggers calibration review), validate_negative_result_dict() (13 required fields guard). CLI (openamp-foundry negative-result-check) with --entry-json, --format text|json. make negative-result-check target. 68 tests. Phase N N4: failed experiments are now documented, not discarded.
 > **New in v0.9.6:** Baseline comparison manifest schema (N3 — Loop 135) — docs/evidence/BASELINE_COMPARISON_GUIDE.md with purpose, schema fields table (14 fields: manifest_id, batch_id, pipeline_version, comparison_date, metric_name, pipeline_score, baseline_scores, pipeline_beats_all_baselines, effect_size, p_value, comparison_direction, notes, reviewer, dry_lab_only), 6 valid metric names (fold_change_mic, hemolysis_fraction, hit_rate, mic_value, novelty_score, selectivity_index), 2 valid comparison directions (higher_is_better, lower_is_better), 11 validation rules, 4 warning conditions (pipeline loses, inconsistent verdict, no p-value, large unchecked effect), honest-use boundary. BaselineComparisonEntry dataclass (14 fields), BaselineComparisonResult dataclass (7 fields), validate_baseline_comparison() (11+ error checks: BCM- prefix, valid metric_name, finite pipeline_score, >=1 baseline, finite baseline_scores, finite effect_size, p_value in [0.0,1.0] or -1.0, valid comparison_direction, notes<=300, non-empty reviewer, dry_lab_only must be True; 4 warning conditions: underperforms, inconsistent verdict, no p-value, large+unchecked), validate_baseline_comparison_dict() (13 required fields guard). CLI (openamp-foundry baseline-comparison-check) with --entry-json, --format text|json. make baseline-comparison-check target. 68 tests. Phase N N3: every performance claim against a baseline is now machine-verifiable.
@@ -131,7 +132,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test).
 > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination.
 > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison.
-> **Pipeline version:** v0.9.2 (M4 — reviewer briefing package)
+> **Pipeline version:** v0.9.9 (O1 — calibration performance summary)
 > **Branch:** main
 
 ---

diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md
@@ -237,6 +237,18 @@ Make experiment predictions falsifiable before results are observed. Guard again
 | N4 | Add negative result record schema (complete — Loop 136). | Ensures failed experiments are documented, not discarded. | B | | Ensures failed experiments are documented, not discarded. | B |
 | N5 | Add experiment priority justification schema (complete — Loop 137). | Documents why this batch was selected over alternatives. Completes Phase N. | B |
 
+## Phase O — Calibration Quality Assurance
+
+Track how well predictions match experimental outcomes over time. Detect drift, overconfidence, and systematic bias before they compound.
+
+| PR | Task | Why it matters | Review class |
+|---:|---|---|---|
+| O1 | Add calibration performance summary schema (complete — Loop 138). | Aggregates prediction accuracy metrics across batches with known outcomes. | B/D |
+| O2 | Add prediction drift monitor schema. | Detects when pipeline predictions are systematically shifting. | B/D |
+| O3 | Add calibration improvement record schema. | Documents what was changed to improve calibration and by how much. | B/D |
+| O4 | Add cross-batch performance aggregator schema. | Aggregates results across batches for trend analysis. | B/D |
+| O5 | Add calibration readiness gate schema. | Validates calibration quality is sufficient before releasing the next batch. | B/D |
+
 ## Prioritization rule
 
 When choosing between tasks, prefer the one that:

diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md
@@ -1,5 +1,12 @@
 # Roadmap
 
+### v0.9.9 — Phase O O1: Calibration Performance Summary (starts Phase O)
+- Added `CalibrationPerformanceEntry` schema tracking prediction accuracy over batches with known outcomes
+- Confusion matrix validation; Brier score; FP rate, recall, and small-sample warnings
+- dry_lab_only=False (requires real experimental data)
+- CLI: `openamp-foundry calibration-performance-check`
+- 46 tests; all passing
+
 ### v0.9.8 — Phase N N5: Experiment Priority Justification (Completes Phase N)
 - Added `ExperimentPriorityEntry` schema documenting why a batch was selected over alternatives
 - Safety review enforcement; post-hoc selection warning; resource constraint transparency

diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py
@@ -3407,3 +3407,22 @@ def _run_experiment_priority_check(args: argparse.Namespace) -> None:
             print(f"  ERROR: {e}")
         for w in result.warnings:
             print(f"  WARN:  {w}")
+
+
+def _run_calibration_performance_check(args: argparse.Namespace) -> None:
+    import json
+    from openamp_foundry.evidence.calibration_performance_summary import validate_calibration_performance_dict
+
+    entry_dict = json.loads(args.entry_json)
+    result = validate_calibration_performance_dict(entry_dict)
+
+    if args.format == "json":
+        import dataclasses
+        print(json.dumps(dataclasses.asdict(result), indent=2))
+    else:
+        status = "PASS" if result.passed else "FAIL"
+        print(f"[{status}] Calibration Performance: {result.summary_id} (v{result.pipeline_version}, n={result.total_candidates_evaluated}, Brier={result.brier_score:.3f})")
+        for e in result.errors:
+            print(f"  ERROR: {e}")
+        for w in result.warnings:
+            print(f"  WARN:  {w}")
diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py
@@ -5,7 +5,7 @@
 from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale
 from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus
 from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc
-from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard, _run_reviewer_briefing_check, _run_audit_chain_check, _run_pre_registration_check, _run_hypothesis_outcome_check, _run_baseline_comparison_check, _run_negative_result_check, _run_experiment_priority_check
+from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard, _run_reviewer_briefing_check, _run_audit_chain_check, _run_pre_registration_check, _run_hypothesis_outcome_check, _run_baseline_comparison_check, _run_negative_result_check, _run_experiment_priority_check, _run_calibration_performance_check
 from openamp_foundry.cli.commands.gates import _run_gate_check, _run_release_gate_check
 from openamp_foundry.cli.commands.reports import _run_contribution_check
 from openamp_foundry.cli.commands.reports import _run_decision_log
@@ -2296,6 +2296,16 @@ def build_parser() -> argparse.ArgumentParser:
     )
     experiment_priority_parser.set_defaults(func=_run_experiment_priority_check)
 
+    calibration_performance_parser = subparsers.add_parser(
+        "calibration-performance-check",
+        help="Validate a calibration performance summary entry",
+    )
+    calibration_performance_parser.add_argument("--entry-json", required=True)
+    calibration_performance_parser.add_argument(
+        "--format", choices=["text", "json"], default="text"
+    )
+    calibration_performance_parser.set_defaults(func=_run_calibration_performance_check)
+
     # ── Selection rationale check (Phase K K1) ───────────────────────
     src2 = sub.add_parser(
         "selection-rationale-check",
@@ -2659,6 +2669,8 @@ def main(argv: list[str] | None = None) -> int:
         return _run_negative_result_check(args)
     if args.command == "experiment-priority-check":
         return _run_experiment_priority_check(args)
+    if args.command == "calibration-performance-check":
+        return _run_calibration_performance_check(args)
 
     parser.error("unknown command")
     return 2