feat: Phase O O2 prediction drift monitor schema -- earliest warning signal for pipeline score distribution shifts

Makefile

@@ -966,6 +966,10 @@ experiment-priority-check:
 calibration-performance-check:
 	openamp-foundry calibration-performance-check --entry-json '{"summary_id":"CPS-001","pipeline_version":"0.9.9","evaluation_date":"2026-07-10","batch_ids_evaluated":["BATCH-001","BATCH-002"],"total_candidates_evaluated":20,"true_positive_count":10,"false_positive_count":2,"true_negative_count":6,"false_negative_count":2,"brier_score":0.12,"calibration_notes":"Calibration looks reasonable.","reviewer":"test@example.com","dry_lab_only":false}' --format text
 
+.PHONY: prediction-drift-check
+prediction-drift-check:
+	openamp-foundry prediction-drift-check --entry-json '{"monitor_id":"DRM-TEST","pipeline_version":"v1.0","reference_batch_id":"A","evaluation_batch_id":"B","reference_mean_score":0.6,"reference_std_score":0.1,"evaluation_mean_score":0.65,"evaluation_std_score":0.12,"mean_shift_magnitude":0.05,"population_size_reference":50,"population_size_evaluation":50,"drift_flag":false,"drift_notes":"","reviewer":"test"}'
+
 dataset-release-check:
 	.venv/bin/openamp-foundry dataset-release-check \
 		--entry-json '{"release_id":"DSR-001","dataset_name":"OpenAMP Wave 0.5 Candidates","dataset_version":"1.0.0","release_date":"2026-07-09","license_identifier":"CC-BY-4.0","data_sources":["UniProt AMP database","APD3","in-house pipeline"],"contains_sequences":true,"contains_activity_data":false,"dual_use_assessed":true,"usage_policy_url":"https://openamp.example.org/data-policy","contact_email":"data@openamp.example.org","release_approved":true,"dry_lab_only":true}' \

docs/evidence/METRICS_CURRENT.md

@@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
 > with this file, this file wins. Updated whenever benchmark/benchmark config changes.
 >
-> **Last updated:** 2026-07-10 (Phase O O1 — calibration performance summary — v0.9.9)
+> **Last updated:** 2026-07-10 (Phase O O2 — prediction drift monitor — v0.10.0)
+> **New in v0.10.0 (Phase O O2):** PredictionDriftEntry schema (15 fields: monitor_id, pipeline_version, reference_batch_id, evaluation_batch_id, reference_mean_score, reference_std_score, evaluation_mean_score, evaluation_std_score, mean_shift_magnitude, population_size_reference, population_size_evaluation, drift_flag, drift_notes, reviewer, dry_lab_only). 8 validation rules (DRM- prefix, batch id must differ, scores 0.0-1.0, std >= 0.0, mean shift tolerance ±0.001, population >=1, notes <=400 chars, drift_flag=True requires non-empty notes). 4 warning conditions (unreported large shift >=0.1, small population <10, variance explosion >2x). 5 constants (DRIFT_NOTES_MAX_LENGTH=400, MEAN_SHIFT_TOLERANCE=0.001, SIGNIFICANT_DRIFT_THRESHOLD=0.1, MIN_POPULATION_FOR_RELIABLE_DRIFT=10, VARIANCE_EXPLOSION_RATIO=2.0). PredictionDriftEntry (15 fields), PredictionDriftResult (8 fields: monitor_id, pipeline_version, mean_shift_magnitude, drift_flag, passed, errors, warnings, dry_lab_only). validate_prediction_drift() (12+ error checks, 4 warning conditions). validate_prediction_drift_dict() (14 required fields guard). CLI (openamp-foundry prediction-drift-check) with --entry-json, --format text|json. make prediction-drift-check target. 68 tests. Phase O O2: prediction drift monitor is the earliest warning signal for pipeline score distribution shifts before wet-lab validation data arrives.
 > **New in v0.9.9:** Calibration performance summary schema (O1 — Loop 138) — docs/evidence/CALIBRATION_PERFORMANCE_GUIDE.md with purpose, schema fields table (13 fields: summary_id, pipeline_version, evaluation_date, batch_ids_evaluated, total_candidates_evaluated, true_positive_count, false_positive_count, true_negative_count, false_negative_count, brier_score, calibration_notes, reviewer, dry_lab_only), 4 derived metrics (precision, recall, specificity, F1), 9 validation rules (CPS- prefix, >=1 batch, >=1 candidates, all counts >=0, total == TP+FP+TN+FN, Brier 0.0-1.0, notes<=400 chars, non-empty reviewer, dry_lab_only must be False), 4 warning conditions (high FP rate >0.5, low recall <0.3, poor Brier >0.25, small sample <10), honest-use boundary. CalibrationPerformanceEntry dataclass (13 fields, dry_lab_only=False), CalibrationPerformanceResult dataclass (8 fields: summary_id, pipeline_version, total_candidates_evaluated, brier_score, passed, errors, warnings, dry_lab_only=False), MAX_CALIBRATION_NOTES_LENGTH=400, MIN_CANDIDATES_FOR_RELIABLE_ESTIMATE=10, HIGH_FP_RATE_THRESHOLD=0.5, LOW_RECALL_THRESHOLD=0.3, POOR_BRIER_SCORE_THRESHOLD=0.25. validate_calibration_performance() (9+ error checks: CPS- prefix, >=1 batch, >=1 candidates, >=0 counts, confusion sum match, Brier bounds, notes<=400, non-empty reviewer, dry_lab_only must be False; 4 warnings: high FP, low recall, poor Brier, small sample). validate_calibration_performance_dict() (12 required fields guard). CLI (openamp-foundry calibration-performance-check) with --entry-json, --format text|json. make calibration-performance-check target. 68 tests. Phase O O1: calibration performance summaries now measure how well predictions match experimental outcomes. Phase O (Calibration Quality Assurance) started.
 > **New in v0.9.8:** Experiment priority justification schema (N5 — Loop 137) — docs/evidence/EXPERIMENT_PRIORITY_GUIDE.md with purpose, schema fields table (12 fields: justification_id, batch_id, pipeline_version, decision_date, selection_criteria, rejected_alternatives, rejection_rationale, resource_constraint, safety_reviewed, pre_specified, decided_by, dry_lab_only), 8 validation rules (EPJ- prefix, >=2 selection criteria, >=1 rejected alternative, non-empty rejection_rationale<=500 chars, resource_constraint<=200 chars, safety_reviewed must be True, non-empty decided_by, dry_lab_only must be True), 4 warning conditions (post-hoc, minimum criteria, no resource constraint, many criteria), honest-use boundary. ExperimentPriorityEntry dataclass (12 fields, dry_lab_only=True), ExperimentPriorityResult dataclass (8 fields: justification_id, batch_id, criteria_count, rejected_alternative_count, safety_reviewed, passed, errors, warnings, dry_lab_only=True), MINIMUM_SELECTION_CRITERIA=2, MAXIMUM_SELECTION_CRITERIA_WARNING=6, MAX_REJECTION_RATIONALE_LENGTH=500, MAX_RESOURCE_CONSTRAINT_LENGTH=200. validate_experiment_priority() (8+ error checks: EPJ- prefix, >=2 criteria, >=1 alternative, non-empty rationale<=500 chars, constraint<=200 chars, safety_reviewed must be True, non-empty decided_by, dry_lab_only must be True; 4 warnings: post-hoc, minimum criteria, no constraint, many criteria). validate_experiment_priority_dict() (11 required fields guard). CLI (openamp-foundry experiment-priority-check) with --entry-json, --format text|json. make experiment-priority-check target. 62 tests. Phase N N5: batch selection decisions are now documented with full audit trail over alternatives. Phase N (Pre-registration & Baseline Honesty) is fully complete with N1-N5.
 > **New in v0.9.7:** Negative result record schema (N4 — Loop 136) — docs/evidence/NEGATIVE_RESULT_GUIDE.md with purpose, schema fields table (14 fields: record_id, batch_id, pipeline_version, record_date, failure_category, failure_description, candidate_ids, assay_type, expected_outcome, observed_outcome, hypothesis_impact, will_be_reported, recorded_by, dry_lab_only), 6 valid failure categories (assay_quality_failure, below_activity_threshold, excessive_toxicity, model_overprediction, pipeline_error, stability_failure), 5 valid assay types (cytotoxicity_assay, hemolysis_assay, membrane_disruption_assay, mic_assay, stability_assay), 9 validation rules, 3 warning conditions (not reported, large failure set, model overprediction without recalibration), honest-use boundary. NegativeResultEntry dataclass (14 fields), NegativeResultResult dataclass (7 fields), validate_negative_result() (9+ error checks: NRR- prefix, valid failure_category, non-empty failure_description<=500 chars, >=1 candidate_ids, valid assay_type, non-empty expected_outcome/observed_outcome, non-empty hypothesis_impact<=300 chars, non-empty recorded_by; 3 warnings: not reported contributes to publication bias, >10 candidates suggests systematic issue, model_overprediction without calibration mention triggers calibration review), validate_negative_result_dict() (13 required fields guard). CLI (openamp-foundry negative-result-check) with --entry-json, --format text|json. make negative-result-check target. 68 tests. Phase N N4: failed experiments are now documented, not discarded.

docs/evidence/PREDICTION_DRIFT_GUIDE.md

@@ -0,0 +1,58 @@
+# Prediction Drift Monitor Guide
+
+Schema: `PredictionDriftEntry` — tracks score distribution shifts across pipeline batches.
+
+## Purpose
+
+Prediction drift monitoring detects when the pipeline's score distributions shift between
+batches without a corresponding change in pipeline version. Drift can indicate:
+- Changes in input sequence distribution (different organisms, assay conditions)
+- Silent model degradation or data leakage
+- Recalibration need before lab results confirm it
+
+This is the earliest warning signal available before wet-lab validation data arrives.
+
+## Fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `monitor_id` | str | Unique ID, must start with "DRM-" |
+| `pipeline_version` | str | Pipeline version being monitored |
+| `reference_batch_id` | str | Batch used as baseline for drift comparison |
+| `evaluation_batch_id` | str | Batch being evaluated for drift |
+| `reference_mean_score` | float | Mean prediction score in reference batch (0.0–1.0) |
+| `reference_std_score` | float | Std dev of scores in reference batch (≥ 0.0) |
+| `evaluation_mean_score` | float | Mean prediction score in evaluation batch (0.0–1.0) |
+| `evaluation_std_score` | float | Std dev of scores in evaluation batch (≥ 0.0) |
+| `mean_shift_magnitude` | float | Absolute difference in means (≥ 0.0) |
+| `population_size_reference` | int | Number of candidates in reference batch (≥ 1) |
+| `population_size_evaluation` | int | Number of candidates in evaluation batch (≥ 1) |
+| `drift_flag` | bool | True if drift exceeds threshold |
+| `drift_notes` | str | Max 400 chars; required when drift_flag=True |
+| `reviewer` | str | Who reviewed this drift assessment |
+| `dry_lab_only` | bool | Always True (computational monitoring) |
+
+## Validation Rules
+
+1. `monitor_id` must start with "DRM-"
+2. `reference_batch_id` and `evaluation_batch_id` must differ
+3. `reference_mean_score` and `evaluation_mean_score` must be in [0.0, 1.0]
+4. `reference_std_score` and `evaluation_std_score` must be ≥ 0.0
+5. `mean_shift_magnitude` must equal |evaluation_mean_score - reference_mean_score| ± 0.001
+6. `population_size_reference` and `population_size_evaluation` must be ≥ 1
+7. `drift_notes` must not exceed 400 characters
+8. When `drift_flag=True`, `drift_notes` must be non-empty
+
+## Warning Conditions
+
+- Mean shift ≥ 0.1 without drift_flag=True (possible unreported drift)
+- Either population < 10 (small sample, drift estimate unreliable)
+- Std dev in evaluation batch > 2× reference std dev (variance explosion)
+
+## Constants
+
+- `DRIFT_NOTES_MAX_LENGTH = 400`
+- `MEAN_SHIFT_TOLERANCE = 0.001`
+- `SIGNIFICANT_DRIFT_THRESHOLD = 0.1`
+- `MIN_POPULATION_FOR_RELIABLE_DRIFT = 10`
+- `VARIANCE_EXPLOSION_RATIO = 2.0`

docs/research/NEXT_100_PR_MAP.md

@@ -244,7 +244,7 @@ Track how well predictions match experimental outcomes over time. Detect drift,
 | PR | Task | Why it matters | Review class |
 |---:|---|---|---|
 | O1 | Add calibration performance summary schema (complete — Loop 138). | Aggregates prediction accuracy metrics across batches with known outcomes. | B/D |
-| O2 | Add prediction drift monitor schema. | Detects when pipeline predictions are systematically shifting. | B/D |
+| O2 | Add prediction drift monitor schema (complete — Loop 139). | Detects when pipeline predictions are systematically shifting. | B/D |
 | O3 | Add calibration improvement record schema. | Documents what was changed to improve calibration and by how much. | B/D |
 | O4 | Add cross-batch performance aggregator schema. | Aggregates results across batches for trend analysis. | B/D |
 | O5 | Add calibration readiness gate schema. | Validates calibration quality is sufficient before releasing the next batch. | B/D |

docs/research/ROADMAP.md

@@ -1,5 +1,12 @@
 # Roadmap
 
+### v0.10.0 — Phase O O2: Prediction Drift Monitor
+- Added `PredictionDriftEntry` schema tracking mean/std score shifts between reference and evaluation batches
+- Detects silent model degradation before wet-lab confirmation
+- Constants: SIGNIFICANT_DRIFT_THRESHOLD=0.1, VARIANCE_EXPLOSION_RATIO=2.0
+- CLI: `openamp-foundry prediction-drift-check`
+- 68 tests; all passing
+
 ### v0.9.9 — Phase O O1: Calibration Performance Summary (starts Phase O)
 - Added `CalibrationPerformanceEntry` schema tracking prediction accuracy over batches with known outcomes
 - Confusion matrix validation; Brier score; FP rate, recall, and small-sample warnings

src/openamp_foundry/cli/commands/reports.py

@@ -3426,3 +3426,34 @@ def _run_calibration_performance_check(args: argparse.Namespace) -> None:
             print(f"  ERROR: {e}")
         for w in result.warnings:
             print(f"  WARN:  {w}")
+
+
+def _run_prediction_drift_check(args) -> int:
+    import json
+    from openamp_foundry.evidence.prediction_drift_monitor import (
+        validate_prediction_drift_dict,
+    )
+
+    if args.entry_json:
+        try:
+            data = json.loads(args.entry_json)
+        except json.JSONDecodeError as exc:
+            print(f"Error: invalid JSON: {exc}", file=__import__("sys").stderr)
+            return 1
+    else:
+        data = json.load(__import__("sys").stdin)
+
+    result = validate_prediction_drift_dict(data)
+
+    if args.format == "json":
+        import dataclasses
+        print(json.dumps(dataclasses.asdict(result), indent=2))
+    else:
+        status = "PASS" if result.passed else "FAIL"
+        print(f"[{status}] Prediction Drift Monitor: {result.monitor_id}")
+        for err in result.errors:
+            print(f"  ERROR: {err}")
+        for warn in result.warnings:
+            print(f"  WARN:  {warn}")
+
+    return 0 if result.passed else 1

src/openamp_foundry/cli/main.py

@@ -5,7 +5,7 @@
 from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale
 from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus
 from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc
-from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard, _run_reviewer_briefing_check, _run_audit_chain_check, _run_pre_registration_check, _run_hypothesis_outcome_check, _run_baseline_comparison_check, _run_negative_result_check, _run_experiment_priority_check, _run_calibration_performance_check
+from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard, _run_reviewer_briefing_check, _run_audit_chain_check, _run_pre_registration_check, _run_hypothesis_outcome_check, _run_baseline_comparison_check, _run_negative_result_check,     _run_experiment_priority_check, _run_calibration_performance_check, _run_prediction_drift_check
 from openamp_foundry.cli.commands.gates import _run_gate_check, _run_release_gate_check
 from openamp_foundry.cli.commands.reports import _run_contribution_check
 from openamp_foundry.cli.commands.reports import _run_decision_log
@@ -2286,7 +2286,7 @@ def build_parser() -> argparse.ArgumentParser:
     )
     negative_result_parser.set_defaults(func=_run_negative_result_check)
 
-    experiment_priority_parser = subparsers.add_parser(
+    experiment_priority_parser = sub.add_parser(
         "experiment-priority-check",
         help="Validate an experiment priority justification entry",
     )
@@ -2296,7 +2296,7 @@ def build_parser() -> argparse.ArgumentParser:
     )
     experiment_priority_parser.set_defaults(func=_run_experiment_priority_check)
 
-    calibration_performance_parser = subparsers.add_parser(
+    calibration_performance_parser = sub.add_parser(
         "calibration-performance-check",
         help="Validate a calibration performance summary entry",
     )
@@ -2306,6 +2306,14 @@ def build_parser() -> argparse.ArgumentParser:
     )
     calibration_performance_parser.set_defaults(func=_run_calibration_performance_check)
 
+    p_drift = sub.add_parser(
+        "prediction-drift-check",
+        help="Validate prediction drift monitor entry",
+    )
+    p_drift.add_argument("--entry-json", default=None)
+    p_drift.add_argument("--format", choices=["text", "json"], default="text")
+    p_drift.set_defaults(func=_run_prediction_drift_check)
+
     # ── Selection rationale check (Phase K K1) ───────────────────────
     src2 = sub.add_parser(
         "selection-rationale-check",
@@ -2672,6 +2680,9 @@ def main(argv: list[str] | None = None) -> int:
     if args.command == "calibration-performance-check":
         return _run_calibration_performance_check(args)
 
+    if args.command == "prediction-drift-check":
+        return _run_prediction_drift_check(args)
+
     parser.error("unknown command")
     return 2
 

src/openamp_foundry/evidence/candidate_summary_card.py

@@ -75,6 +75,12 @@ def validate_candidate_summary_card(
 
     if not entry.sequence:
         errors.append("sequence must not be empty")
+
+    elif entry.sequence != entry.sequence.upper():
+        errors.append(
+            "sequence must be uppercase, got lowercase characters"
+        )
+
     else:
         invalid_chars = set(entry.sequence.upper()) - VALID_AMINO_ACIDS
         if invalid_chars: