feat: Phase O O3 calibration improvement record schema -- auditable history of recalibration actions linking detection to correction

Makefile

@@ -974,3 +974,7 @@ dataset-release-check:
 	.venv/bin/openamp-foundry dataset-release-check \
 		--entry-json '{"release_id":"DSR-001","dataset_name":"OpenAMP Wave 0.5 Candidates","dataset_version":"1.0.0","release_date":"2026-07-09","license_identifier":"CC-BY-4.0","data_sources":["UniProt AMP database","APD3","in-house pipeline"],"contains_sequences":true,"contains_activity_data":false,"dual_use_assessed":true,"usage_policy_url":"https://openamp.example.org/data-policy","contact_email":"data@openamp.example.org","release_approved":true,"dry_lab_only":true}' \
 		--format text
+
+.PHONY: calibration-improvement-check
+calibration-improvement-check:
+	openamp-foundry calibration-improvement-check --entry-json '{"improvement_id":"CIR-TEST","pipeline_version_before":"v1.0","pipeline_version_after":"v1.1","trigger_ids":["CPS-001"],"action_taken":"threshold adjustment","action_category":"threshold_adjustment","brier_score_before":0.3,"brier_score_after":0.2,"improvement_confirmed":true,"reviewer":"test"}'

docs/evidence/CALIBRATION_IMPROVEMENT_GUIDE.md

@@ -0,0 +1,49 @@
+# Calibration Improvement Record Guide
+
+Schema: `CalibrationImprovementEntry` — documents recalibration actions taken after drift or quality failures.
+
+## Purpose
+
+When calibration performance drops below threshold (high FP rate, low recall, poor Brier score) or
+prediction drift is detected, a recalibration action must be taken. This schema provides an auditable
+record of what changed, why, and what effect it had — closing the loop between detection and correction.
+
+## Fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `improvement_id` | str | Unique ID, must start with "CIR-" |
+| `pipeline_version_before` | str | Pipeline version before recalibration |
+| `pipeline_version_after` | str | Pipeline version after recalibration |
+| `trigger_ids` | List[str] | IDs of CPS- or DRM- records that triggered this action (at least 1) |
+| `action_taken` | str | Description of what was changed (max 500 chars) |
+| `action_category` | str | One of: threshold_adjustment, retraining, feature_removal, feature_addition, data_augmentation, weighting_change |
+| `brier_score_before` | float | Brier score before action (0.0–1.0) |
+| `brier_score_after` | float | Brier score after action (0.0–1.0) |
+| `improvement_confirmed` | bool | True if brier_score_after < brier_score_before |
+| `reviewer` | str | Who reviewed this improvement record |
+| `dry_lab_only` | bool | False (may incorporate real lab feedback) |
+
+## Validation Rules
+
+1. `improvement_id` must start with "CIR-"
+2. `pipeline_version_before` and `pipeline_version_after` must differ
+3. `trigger_ids` must contain at least 1 entry
+4. Each trigger_id must start with "CPS-" or "DRM-"
+5. `action_taken` must not exceed 500 characters
+6. `action_category` must be one of the 6 valid categories
+7. `brier_score_before` and `brier_score_after` must be in [0.0, 1.0]
+8. `improvement_confirmed` must be consistent: True iff brier_score_after < brier_score_before
+
+## Warning Conditions
+
+- brier_score_after >= brier_score_before but improvement_confirmed=True (inconsistency)
+- brier_score_after > POOR_BRIER_THRESHOLD (0.25) even after recalibration
+- pipeline_version_before == pipeline_version_after warns about missing version bump
+
+## Constants
+
+- `ACTION_TAKEN_MAX_LENGTH = 500`
+- `VALID_ACTION_CATEGORIES = {"threshold_adjustment", "retraining", "feature_removal", "feature_addition", "data_augmentation", "weighting_change"}`
+- `VALID_TRIGGER_PREFIXES = {"CPS-", "DRM-"}`
+- `POOR_BRIER_THRESHOLD = 0.25`

docs/research/ROADMAP.md

@@ -1,5 +1,10 @@
 # Roadmap
 
+### v0.10.1 - Phase O O3: Calibration Improvement Record
+- CalibrationImprovementEntry schema documents recalibration actions taken after drift/quality failures
+- 6 action categories, trigger_ids linked to CPS-/DRM- records
+- Closes the loop between detection and correction
+
 ### v0.10.0 — Phase O O2: Prediction Drift Monitor
 - Added `PredictionDriftEntry` schema tracking mean/std score shifts between reference and evaluation batches
 - Detects silent model degradation before wet-lab confirmation

src/openamp_foundry/cli/commands/reports.py

@@ -3457,3 +3457,34 @@ def _run_prediction_drift_check(args) -> int:
             print(f"  WARN:  {warn}")
 
     return 0 if result.passed else 1
+
+
+def _run_calibration_improvement_check(args) -> int:
+    import json
+    from openamp_foundry.evidence.calibration_improvement_record import (
+        validate_calibration_improvement_dict,
+    )
+
+    if args.entry_json:
+        try:
+            data = json.loads(args.entry_json)
+        except json.JSONDecodeError as exc:
+            print(f"Error: invalid JSON: {exc}", file=__import__("sys").stderr)
+            return 1
+    else:
+        data = json.load(__import__("sys").stdin)
+
+    result = validate_calibration_improvement_dict(data)
+
+    if args.format == "json":
+        import dataclasses
+        print(json.dumps(dataclasses.asdict(result), indent=2))
+    else:
+        status = "PASS" if result.passed else "FAIL"
+        print(f"[{status}] Calibration Improvement: {result.improvement_id}")
+        for err in result.errors:
+            print(f"  ERROR: {err}")
+        for warn in result.warnings:
+            print(f"  WARN:  {warn}")
+
+    return 0 if result.passed else 1

src/openamp_foundry/cli/main.py

@@ -5,7 +5,23 @@
 from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale
 from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus
 from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc
-from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard, _run_reviewer_briefing_check, _run_audit_chain_check, _run_pre_registration_check, _run_hypothesis_outcome_check, _run_baseline_comparison_check, _run_negative_result_check,     _run_experiment_priority_check, _run_calibration_performance_check, _run_prediction_drift_check
+from openamp_foundry.cli.commands.reports import (
+    _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard,
+    _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake,
+    _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version,
+    _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter,
+    _run_synthetic_result_policy_check, _run_calibration_decision_checklist,
+    _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result,
+    _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance,
+    _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check,
+    _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version,
+    _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check,
+    _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard,
+    _run_reviewer_briefing_check, _run_audit_chain_check, _run_pre_registration_check,
+    _run_hypothesis_outcome_check, _run_baseline_comparison_check, _run_negative_result_check,
+    _run_experiment_priority_check, _run_calibration_performance_check, _run_prediction_drift_check,
+    _run_calibration_improvement_check,
+)
 from openamp_foundry.cli.commands.gates import _run_gate_check, _run_release_gate_check
 from openamp_foundry.cli.commands.reports import _run_contribution_check
 from openamp_foundry.cli.commands.reports import _run_decision_log
@@ -2314,6 +2330,14 @@ def build_parser() -> argparse.ArgumentParser:
     p_drift.add_argument("--format", choices=["text", "json"], default="text")
     p_drift.set_defaults(func=_run_prediction_drift_check)
 
+    p_cir = sub.add_parser(
+        "calibration-improvement-check",
+        help="Validate calibration improvement record entry",
+    )
+    p_cir.add_argument("--entry-json", default=None)
+    p_cir.add_argument("--format", choices=["text", "json"], default="text")
+    p_cir.set_defaults(func=_run_calibration_improvement_check)
+
     # ── Selection rationale check (Phase K K1) ───────────────────────
     src2 = sub.add_parser(
         "selection-rationale-check",

src/openamp_foundry/evidence/calibration_improvement_record.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Set
+
+ACTION_TAKEN_MAX_LENGTH: int = 500
+VALID_ACTION_CATEGORIES: Set[str] = {
+    "threshold_adjustment",
+    "retraining",
+    "feature_removal",
+    "feature_addition",
+    "data_augmentation",
+    "weighting_change",
+}
+VALID_TRIGGER_PREFIXES: Set[str] = {"CPS-", "DRM-"}
+POOR_BRIER_THRESHOLD: float = 0.25
+
+
+@dataclass
+class CalibrationImprovementEntry:
+    improvement_id: str
+    pipeline_version_before: str
+    pipeline_version_after: str
+    trigger_ids: List[str]
+    action_taken: str
+    action_category: str
+    brier_score_before: float
+    brier_score_after: float
+    improvement_confirmed: bool
+    reviewer: str
+    dry_lab_only: bool = False
+
+
+@dataclass
+class CalibrationImprovementResult:
+    improvement_id: str
+    pipeline_version_before: str
+    pipeline_version_after: str
+    brier_score_before: float
+    brier_score_after: float
+    improvement_confirmed: bool
+    passed: bool
+    errors: List[str] = field(default_factory=list)
+    warnings: List[str] = field(default_factory=list)
+    dry_lab_only: bool = False
+
+
+def validate_calibration_improvement(
+    entry: CalibrationImprovementEntry,
+) -> CalibrationImprovementResult:
+    errors: List[str] = []
+    warnings: List[str] = []
+
+    if not entry.improvement_id.startswith("CIR-"):
+        errors.append(
+            f"improvement_id must start with 'CIR-', got '{entry.improvement_id}'"
+        )
+
+    if entry.pipeline_version_before == entry.pipeline_version_after:
+        errors.append(
+            "pipeline_version_before and pipeline_version_after must differ"
+        )
+
+    if not entry.trigger_ids:
+        errors.append("trigger_ids must contain at least 1 entry")
+    else:
+        bad_triggers = [
+            tid for tid in entry.trigger_ids
+            if not any(tid.startswith(p) for p in VALID_TRIGGER_PREFIXES)
+        ]
+        if bad_triggers:
+            errors.append(
+                f"trigger_ids must start with CPS- or DRM-: {bad_triggers}"
+            )
+
+    if len(entry.action_taken) > ACTION_TAKEN_MAX_LENGTH:
+        errors.append(
+            f"action_taken exceeds {ACTION_TAKEN_MAX_LENGTH} chars "
+            f"(got {len(entry.action_taken)})"
+        )
+
+    if entry.action_category not in VALID_ACTION_CATEGORIES:
+        errors.append(
+            f"action_category must be one of {sorted(VALID_ACTION_CATEGORIES)}, "
+            f"got '{entry.action_category}'"
+        )
+
+    if not (0.0 <= entry.brier_score_before <= 1.0):
+        errors.append(
+            f"brier_score_before must be in [0.0, 1.0], "
+            f"got {entry.brier_score_before}"
+        )
+
+    if not (0.0 <= entry.brier_score_after <= 1.0):
+        errors.append(
+            f"brier_score_after must be in [0.0, 1.0], "
+            f"got {entry.brier_score_after}"
+        )
+
+    expected_confirmed = entry.brier_score_after < entry.brier_score_before
+    if entry.improvement_confirmed != expected_confirmed:
+        errors.append(
+            f"improvement_confirmed ({entry.improvement_confirmed}) is inconsistent: "
+            f"brier_score_after ({entry.brier_score_after}) "
+            f"{'<' if expected_confirmed else '>='} "
+            f"brier_score_before ({entry.brier_score_before})"
+        )
+
+    # Warnings
+    if entry.brier_score_after >= POOR_BRIER_THRESHOLD:
+        warnings.append(
+            f"brier_score_after ({entry.brier_score_after}) >= "
+            f"{POOR_BRIER_THRESHOLD} — calibration still poor after action"
+        )
+
+    return CalibrationImprovementResult(
+        improvement_id=entry.improvement_id,
+        pipeline_version_before=entry.pipeline_version_before,
+        pipeline_version_after=entry.pipeline_version_after,
+        brier_score_before=entry.brier_score_before,
+        brier_score_after=entry.brier_score_after,
+        improvement_confirmed=entry.improvement_confirmed,
+        passed=len(errors) == 0,
+        errors=errors,
+        warnings=warnings,
+        dry_lab_only=entry.dry_lab_only,
+    )
+
+
+def validate_calibration_improvement_dict(data: dict) -> CalibrationImprovementResult:
+    required = [
+        "improvement_id", "pipeline_version_before", "pipeline_version_after",
+        "trigger_ids", "action_taken", "action_category",
+        "brier_score_before", "brier_score_after", "improvement_confirmed", "reviewer",
+    ]
+    errors: List[str] = []
+    for key in required:
+        if key not in data:
+            errors.append(f"Missing required field: {key}")
+    if errors:
+        return CalibrationImprovementResult(
+            improvement_id=data.get("improvement_id", ""),
+            pipeline_version_before=data.get("pipeline_version_before", ""),
+            pipeline_version_after=data.get("pipeline_version_after", ""),
+            brier_score_before=float(data.get("brier_score_before", 0.0)),
+            brier_score_after=float(data.get("brier_score_after", 0.0)),
+            improvement_confirmed=bool(data.get("improvement_confirmed", False)),
+            passed=False,
+            errors=errors,
+            warnings=[],
+        )
+    entry = CalibrationImprovementEntry(
+        improvement_id=str(data["improvement_id"]),
+        pipeline_version_before=str(data["pipeline_version_before"]),
+        pipeline_version_after=str(data["pipeline_version_after"]),
+        trigger_ids=list(data["trigger_ids"]),
+        action_taken=str(data["action_taken"]),
+        action_category=str(data["action_category"]),
+        brier_score_before=float(data["brier_score_before"]),
+        brier_score_after=float(data["brier_score_after"]),
+        improvement_confirmed=bool(data["improvement_confirmed"]),
+        reviewer=str(data["reviewer"]),
+        dry_lab_only=bool(data.get("dry_lab_only", False)),
+    )
+    return validate_calibration_improvement(entry)