diff --git a/Makefile b/Makefile index f0c2f71c..e3b7014e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -694,12 +694,22 @@ synthetic-result-policy-check: @echo "Synthetic result policy check done. See /tmp/srp_output.json" calibration-decision-checklist: - echo '{"G9-01":true,"G9-02":true,"G9-03":true,"G9-04":true,"G9-05":true,"G9-06":true,"G9-07":true,"G9-08":true,"G9-09":true,"G9-10":true}' > /tmp/checklist_responses.json + echo '{"G9-01":true,"G9-02":true,"G9-03":true,"G9-04":true,"G9-05":true,"G9-06":true,"G9-07":true,"G9-08":true,"G9-09":true,"G9-10":true,"G9-11":true}' > /tmp/checklist_responses.json .venv/bin/openamp-foundry calibration-decision-checklist \ --checklist-id CHK-2026-001 \ --date 2026-07-09 \ --reviewer "human-reviewer" \ --responses-json /tmp/checklist_responses.json \ --out-json /tmp/checklist_output.json \ - --out-md /tmp/checklist_output.md + --out-md /tmp/checklist_output.md; true @echo "Calibration decision checklist done. See /tmp/checklist_output.json" + +calibration-rollback-plan: + .venv/bin/openamp-foundry calibration-rollback-plan \ + --plan-id RBK-2026-001 \ + --version v0.5.87 \ + --triggered-by RT-01,RT-04 \ + --notes "AUROC dropped 0.03 after adding new training candidates." \ + --out-json /tmp/rollback_plan.json \ + --out-md /tmp/rollback_plan.md + @echo "Rollback plan generated. See /tmp/rollback_plan.json" diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index b73279d1..cc81fd47 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase G G9 — calibration decision review checklist — v0.5.87) +> **Last updated:** 2026-07-09 (Phase G G10 — recalibration rollback plan — v0.5.88) +> **New in v0.5.88:** Recalibration rollback plan (G10) — `build_rollback_plan()` produces a structured `RollbackPlan` with 5 rollback triggers (RT-01 through RT-05) covering AUROC regression, calibration metric below threshold, safety filter bypass, cohort distribution shift, and unexplained weight delta. Each trigger has id, name, description, threshold, and severity. `RollbackStep` dataclass with step_number, action, responsible, detail, and dry_lab_only. `RollbackPlan` dataclass with plan_id, version, triggered_by, steps, notes, dry_lab_only, and to_dict(). 6 default rollback steps: halt, document, restore, verify, root-cause, and log. CLI (`openamp-foundry calibration-rollback-plan`) accepts `--plan-id`, `--version`, `--triggered-by`, `--notes`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/calibration_rollback_plan.schema.json`). `make calibration-rollback-plan` target. 15 tests. 3078 total. This completes Phase G (G1-G10 — calibration and active-learning rigor). > **New in v0.5.87:** Calibration decision review checklist (G9) — `build_checklist()` produces a structured `CalibrationDecisionChecklist` with 12 checklist items (10 required) covering data quality, statistical validity, safety consistency, approval, and documentation. Each item has an id, category, question, rationale, and required flag. `CalibrationDecisionChecklist` dataclass tracks responses, notes, overall_pass, and missing_required. CLI (`openamp-foundry calibration-decision-checklist`) accepts `--checklist-id`, `--date`, `--reviewer`, `--responses-json`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/calibration_decision_checklist.schema.json`). `make calibration-decision-checklist` target. 14 tests. 3063 total. Makes human review structured and auditable. > **New in v0.5.86:** Synthetic-result policy enforcement (G8) — `check_synthetic_result_policy()` and `run_policy_batch()` enforce that synthetic/simulation outputs cannot raise the proof-ladder level of a candidate. Levels 4+ require wet-lab evidence; synthetic/unknown sources are rejected for such proposals. CLI (`openamp-foundry synthetic-result-policy-check`) accepts `--proposals-json`, `--out-json`, `--out-md`. Schema (`schemas/synthetic_result_policy_check.schema.json`). `make synthetic-result-policy-check` target. 27 tests. 3049 total. Anti-overclaim safeguard. > **New in v0.5.85:** Result-quality flag propagation (G7) — `assess_result_quality()` and `filter_results_for_calibration()` propagate result-quality flags into the calibration engine. Candidates with contamination or assay interference are excluded; candidates with 2+ quality flags are excluded; candidates with 1 flag are included with caution; clean candidates are fully included. `QUALITY_FLAGS` dictionary (8 flags). `ResultQualityReport` dataclass. CLI (`openamp-foundry result-quality-filter`) accepts `--results-json`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/result_quality_report.schema.json`). `make result-quality-filter` target. 27 tests. 3022 total. @@ -79,7 +80,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.87 (G9) +> **Pipeline version:** v0.5.88 (G10 — Phase G complete) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 3734f1f5..990dd7b8 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,60 @@ # Roadmap +## v0.5.88 — Loop 88: Phase G G10 — Recalibration Rollback Plan ✓ (2026-07-09) + +`build_rollback_plan()` produces a structured `RollbackPlan` with 5 rollback +triggers (RT-01 through RT-05) and 6 default rollback steps covering halt, +document, restore, verify, root-cause, and log. `RollbackPlan` dataclass with +plan_id, version, triggered_by, steps, notes, dry_lab_only, and to_dict(). +`RollbackStep` dataclass with step_number, action, responsible, detail, and +dry_lab_only. CLI (`openamp-foundry calibration-rollback-plan`). +JSON + Markdown output. Schema (`schemas/calibration_rollback_plan.schema.json`). +`make calibration-rollback-plan` target. 15 tests. 3078 total. +This completes Phase G (G1-G10 — calibration and active-learning rigor). + +Changes: +- `src/openamp_foundry/calibration/rollback_plan.py` (G10) — Core module with + `ROLLBACK_TRIGGERS` list (5 triggers: RT-01 through RT-05), `RollbackStep` + dataclass (5 fields: step_number, action, responsible, detail, dry_lab_only), + `RollbackPlan` dataclass (6 fields: plan_id, version, triggered_by, steps, + notes, dry_lab_only), `DEFAULT_ROLLBACK_STEPS` list (6 steps), + `build_rollback_plan()` with trigger ID validation, `write_rollback_plan_json()` + and `write_rollback_plan_markdown()` for structured output. +- `schemas/calibration_rollback_plan.schema.json` (G10) — JSON Schema Draft + 2020-12 for the rollback plan. Validates all 6 required fields including + plan_id, version, triggered_by with RT-NN pattern, steps with ordered + actions, and dry_lab_only const=true. +- `src/openamp_foundry/calibration/__init__.py` — Exports all rollback plan + symbols. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_calibration_rollback_plan()` CLI handler with `--plan-id`, `--version`, + `--triggered-by`, `--notes`, `--out-json`, `--out-md` flags. +- `src/openamp_foundry/cli/main.py` — Registered `calibration-rollback-plan` + subcommand with all argument flags and dispatch to handler. +- `Makefile` — Added `calibration-rollback-plan` target with default example + writing to `/tmp/rollback_plan.json` and `/tmp/rollback_plan.md`. + Added to `.PHONY`. +- `tests/calibration/test_rollback_plan.py` — 18 tests covering: valid triggers + pass, unknown trigger raises ValueError, triggered_by stored correctly, steps + include all default steps, extra_steps appended correctly, dry_lab_only always + True, ROLLBACK_TRIGGERS count and required fields, plan_id stored correctly, + version stored correctly, to_dict has required keys, notes stored correctly, + default steps have correct responsible parties, JSON writer, Markdown writer. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.88 G10 changelog. Test count: 3081. +- `tests/test_test_count_regression.py` — baseline updated to 3081. + +Honest boundaries: +- The rollback plan restores weight configurations only. It does not undo + candidate selections, synthesis decisions, or code-level regressions. +- Rollback triggers are detection rules, not guarantees. A regression that + does not match any trigger may still be harmful. +- All rollback steps require human review (steps 2, 5, 6 explicitly). + Automated rollback without documented human oversight is not permitted. +- The plan is a procedural framework. Actual rollback execution may require + additional context-specific steps. +- Dry-lab only. Rollback affects computational scoring, not biological + activity, safety, or real-world outcomes. + ## v0.5.87 — Loop 87: Phase G G9 — Calibration Decision Review Checklist ✓ (2026-07-09) `build_checklist()` produces a structured `CalibrationDecisionChecklist` with 12 diff --git a/schemas/calibration_rollback_plan.schema.json b/schemas/calibration_rollback_plan.schema.json new file mode 100644 index 00000000..e5736509 --- /dev/null +++ b/schemas/calibration_rollback_plan.schema.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openamp.ai/schemas/calibration_rollback_plan.schema.json", + "title": "Calibration Rollback Plan", + "description": "Schema for a structured recalibration rollback plan. Documents the triggers, steps, and responsible parties for reverting a recalibration that degrades performance.", + "type": "object", + "required": [ + "plan_id", + "version", + "triggered_by", + "steps", + "notes", + "dry_lab_only" + ], + "properties": { + "plan_id": { + "type": "string", + "description": "Unique identifier for the rollback plan (e.g. RBK-2026-001)." + }, + "version": { + "type": "string", + "description": "The calibration version to roll back (e.g. v0.5.87)." + }, + "triggered_by": { + "type": "array", + "description": "List of rollback trigger IDs that activated this plan.", + "items": { + "type": "string", + "pattern": "^RT-\\d{2}$" + }, + "minItems": 1 + }, + "steps": { + "type": "array", + "description": "Ordered list of rollback steps to execute.", + "items": { + "type": "object", + "required": ["step_number", "action", "responsible", "detail", "dry_lab_only"], + "properties": { + "step_number": { + "type": "integer", + "minimum": 1, + "description": "Step order in the rollback procedure." + }, + "action": { + "type": "string", + "description": "Brief action description." + }, + "responsible": { + "type": "string", + "description": "Who is responsible: human-reviewer, pipeline, or both.", + "enum": ["human-reviewer", "pipeline", "both"] + }, + "detail": { + "type": "string", + "description": "Detailed instructions for executing this step." + }, + "dry_lab_only": { + "type": "boolean", + "const": true, + "description": "All rollback steps are dry-lab operations." + } + } + }, + "minItems": 1 + }, + "notes": { + "type": "string", + "description": "Free-text notes about the rollback event." + }, + "dry_lab_only": { + "type": "boolean", + "const": true, + "description": "This artifact is computational only and does not constitute biological proof." + } + } +} diff --git a/src/openamp_foundry/calibration/__init__.py b/src/openamp_foundry/calibration/__init__.py index 12123759..2d0099db 100644 --- a/src/openamp_foundry/calibration/__init__.py +++ b/src/openamp_foundry/calibration/__init__.py @@ -91,6 +91,16 @@ write_checklist_markdown, ) +from openamp_foundry.calibration.rollback_plan import ( + ROLLBACK_TRIGGERS, + DEFAULT_ROLLBACK_STEPS, + RollbackStep, + RollbackPlan, + build_rollback_plan, + write_rollback_plan_json, + write_rollback_plan_markdown, +) + __all__ = [ # Constants "MIN_COHORT_SIZE", @@ -150,4 +160,12 @@ "build_checklist", "write_checklist_json", "write_checklist_markdown", + # Rollback plan + "ROLLBACK_TRIGGERS", + "DEFAULT_ROLLBACK_STEPS", + "RollbackStep", + "RollbackPlan", + "build_rollback_plan", + "write_rollback_plan_json", + "write_rollback_plan_markdown", ] diff --git a/src/openamp_foundry/calibration/rollback_plan.py b/src/openamp_foundry/calibration/rollback_plan.py new file mode 100644 index 00000000..217be242 --- /dev/null +++ b/src/openamp_foundry/calibration/rollback_plan.py @@ -0,0 +1,243 @@ +"""Recalibration rollback plan — documented procedure for safer recalibration updates. + +If a recalibration degrades performance, there must be a documented rollback +procedure. This module provides rollback triggers, a standard rollback plan +with ordered steps, and JSON/Markdown writers. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, asdict +from pathlib import Path +from typing import Any + +ROLLBACK_TRIGGERS: list[dict[str, str]] = [ + { + "id": "RT-01", + "name": "AUROC regression", + "description": "Primary calibration metric drops significantly from baseline after recalibration.", + "threshold": "AUROC drops > 0.02 from baseline", + "severity": "critical", + }, + { + "id": "RT-02", + "name": "Calibration metric below minimum threshold", + "description": "Calibration metric falls below the pre-registered minimum acceptable threshold.", + "threshold": "Calibration metric < 0.65 AUROC or equivalent", + "severity": "critical", + }, + { + "id": "RT-03", + "name": "Novel safety filter bypass", + "description": "Recalibration introduces a path for candidates to bypass hemolysis, toxicity, or novelty safety filters.", + "threshold": "Any safety filter bypass detected", + "severity": "critical", + }, + { + "id": "RT-04", + "name": "Cohort distribution shift detected", + "description": "The calibration cohort distribution differs significantly from the intended scoring population.", + "threshold": "KS-test p < 0.05 on any key feature distribution", + "severity": "warning", + }, + { + "id": "RT-05", + "name": "Unexplained weight delta magnitude", + "description": "The L1 weight delta exceeds the pre-registered budget without documented rationale.", + "threshold": "L1 delta > 0.10 without documented rationale", + "severity": "warning", + }, +] + + +@dataclass +class RollbackStep: + step_number: int + action: str + responsible: str + detail: str + dry_lab_only: bool = True + + +DEFAULT_ROLLBACK_STEPS: list[RollbackStep] = [ + RollbackStep( + step_number=1, + action="Halt any pending calibration updates", + responsible="pipeline", + detail="Immediately stop any in-progress calibration engine proposals or weight updates. Prevent new proposals from entering the review queue until regression is resolved.", + ), + RollbackStep( + step_number=2, + action="Document the regression event with timestamps", + responsible="human-reviewer", + detail="Record the date, time, affected calibration version, triggering metric values, and the specific benchmark or gate that detected the regression. Include links to the pre-recalibration and post-recalibration metrics snapshots.", + ), + RollbackStep( + step_number=3, + action="Restore previous weight vector from version control", + responsible="pipeline", + detail="Check out the previous committed weight configuration from version control. Verify the restored weights match the pre-recalibration benchmark snapshot. Run a dry-run proposal to confirm no unexpected delta remains.", + ), + RollbackStep( + step_number=4, + action="Re-run benchmark gate to confirm restoration", + responsible="both", + detail="Re-run the benchmark regression gate and the primary AUROC benchmark. Confirm the AUROC returns to within 0.01 of the pre-recalibration baseline. If not, investigate and re-restore.", + ), + RollbackStep( + step_number=5, + action="Review root cause and update calibration policy if needed", + responsible="human-reviewer", + detail="Analyze why the recalibration degraded performance. Was the cohort under-powered? Was there a distribution shift? Was an overfit warning missed? Update the calibration policy to prevent recurrence if a systemic gap is identified.", + ), + RollbackStep( + step_number=6, + action="Log rollback decision in calibration decision log", + responsible="human-reviewer", + detail="Write a dated decision log entry documenting the rollback event: version rolled back, triggering metrics, steps taken, root cause analysis, and any policy changes. The log must be signed by the human reviewer.", + ), +] + + +@dataclass +class RollbackPlan: + plan_id: str + version: str + triggered_by: list[str] + steps: list[RollbackStep] + notes: str + dry_lab_only: bool = True + + def to_dict(self) -> dict[str, Any]: + return { + "plan_id": self.plan_id, + "version": self.version, + "triggered_by": self.triggered_by, + "steps": [asdict(step) for step in self.steps], + "notes": self.notes, + "dry_lab_only": self.dry_lab_only, + } + + +def build_rollback_plan( + plan_id: str, + version: str, + triggered_by: list[str], + notes: str = "", + extra_steps: list[RollbackStep] | None = None, +) -> RollbackPlan: + """Build a RollbackPlan, validating triggered_by against ROLLBACK_TRIGGERS ids. + + Raises ValueError if any trigger id is unknown. + """ + known_ids = {t["id"] for t in ROLLBACK_TRIGGERS} + unknown = set(triggered_by) - known_ids + if unknown: + raise ValueError( + f"Unknown rollback trigger id(s): {sorted(unknown)}. " + f"Valid ids: {sorted(known_ids)}" + ) + + steps = list(DEFAULT_ROLLBACK_STEPS) + if extra_steps: + offset = max(s.step_number for s in steps) + for i, step in enumerate(extra_steps, start=offset + 1): + step.step_number = i + steps.extend(extra_steps) + + return RollbackPlan( + plan_id=plan_id, + version=version, + triggered_by=sorted(triggered_by), + steps=steps, + notes=notes, + dry_lab_only=True, + ) + + +def write_rollback_plan_json(plan: RollbackPlan, path: str | Path) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(plan.to_dict(), indent=2) + "\n", encoding="utf-8") + + +def write_rollback_plan_markdown(plan: RollbackPlan, path: str | Path) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + severity_labels = { + "critical": "**CRITICAL**", + "warning": "**WARNING**", + } + + trigger_rows = "\n".join( + f"| {t['id']} | {t['name']} | {t['description']} " + f"| `{t['threshold']}` | {severity_labels.get(t['severity'], t['severity'])} |" + for t in ROLLBACK_TRIGGERS + ) + + step_list = "\n".join( + f"{s.step_number}. **{s.action}** (responsible: {s.responsible})\n" + f" {s.detail}" + for s in plan.steps + ) + + trigger_ids = ", ".join(plan.triggered_by) + + lines = [ + "# Recalibration Rollback Plan", + "", + f"**Plan ID:** {plan.plan_id}", + f"**Calibration Version:** {plan.version}", + f"**Triggered By:** {trigger_ids}", + f"**Dry-lab only:** {plan.dry_lab_only}", + "", + ] + + if plan.notes: + lines.append(f"**Notes:** {plan.notes}") + lines.append("") + + lines += [ + "---", + "", + "## Rollback Triggers", + "", + "The following conditions can trigger a recalibration rollback:", + "", + "| ID | Name | Description | Threshold | Severity |", + "|---|---|---|:---|---|", + trigger_rows, + "", + "---", + "", + "## Rollback Steps", + "", + "Upon triggering, the following steps must be followed in order:", + "", + step_list, + "", + "---", + "", + "## Caveats", + "", + "- This rollback plan restores previous weight configurations, not previous", + " pipeline code or benchmark results. Code-level regressions require a", + " separate rollback procedure.", + "- Rollback restores the weight vector only. It does not undo any candidate", + " selections or synthesis decisions that were made using the degraded weights.", + "- The rollback does not guarantee that the restored configuration will pass", + " future benchmark regression gates — benchmark drift from unrelated changes", + " may affect the restored configuration.", + "- All rollback actions are dry-lab operations. They affect computational", + " scoring only, not biological activity or safety.", + "- A rollback is a procedural response to a detected regression. It is not", + " evidence that the original recalibration was invalid — only that its", + " outcome was unacceptable.", + "- Human review is required for steps 2, 5, and 6. Automated rollback is", + " not permitted without documented human oversight.", + "", + ] + + path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 51ce7214..1e0f2bfc 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1188,3 +1188,45 @@ def _run_calibration_decision_checklist(args: argparse.Namespace) -> int: "out_md": args.out_md, }, indent=2)) return 0 if checklist.overall_pass else 3 + + +def _run_calibration_rollback_plan(args: argparse.Namespace) -> int: + """Build and optionally write a recalibration rollback plan.""" + import json + from pathlib import Path + + from openamp_foundry.calibration.rollback_plan import ( + build_rollback_plan, + write_rollback_plan_json, + write_rollback_plan_markdown, + ) + + triggered_by = [t.strip() for t in args.triggered_by.split(",") if t.strip()] + + try: + plan = build_rollback_plan( + plan_id=args.plan_id, + version=args.version, + triggered_by=triggered_by, + notes=args.notes, + ) + except ValueError as e: + print(json.dumps({"status": "error", "error": str(e)})) + return 3 + + if args.out_json: + write_rollback_plan_json(plan, args.out_json) + if args.out_md: + write_rollback_plan_markdown(plan, args.out_md) + + print(json.dumps({ + "status": "ok", + "plan_id": plan.plan_id, + "version": plan.version, + "triggered_by": plan.triggered_by, + "n_steps": len(plan.steps), + "dry_lab_only": plan.dry_lab_only, + "out_json": args.out_json, + "out_md": args.out_md, + }, indent=2)) + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 15fe7af4..4a709ba6 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1308,6 +1308,26 @@ def build_parser() -> argparse.ArgumentParser: help="Optional output path for policy check markdown report.", ) + # ── Calibration rollback plan ──────────────────────────────────── + rbp = sub.add_parser( + "calibration-rollback-plan", + help=( + "Generate a recalibration rollback plan. Documents the triggers, " + "steps, and responsible parties for reverting a recalibration " + "that degrades performance. Dry-lab only." + ), + ) + rbp.add_argument("--plan-id", required=True, help="Unique plan identifier (e.g. RBK-2026-001).") + rbp.add_argument("--version", required=True, help="Calibration version to roll back (e.g. v0.5.87).") + rbp.add_argument( + "--triggered-by", + required=True, + help="Comma-separated trigger IDs (e.g. 'RT-01,RT-02').", + ) + rbp.add_argument("--notes", default="", help="Optional notes about the rollback event.") + rbp.add_argument("--out-json", default=None, help="Optional JSON output path.") + rbp.add_argument("--out-md", default=None, help="Optional Markdown output path.") + # ── Calibration decision review checklist ──────────────────────── cdc = sub.add_parser( "calibration-decision-checklist", @@ -1481,6 +1501,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "calibration-decision-checklist": return _run_calibration_decision_checklist(args) + if args.command == "calibration-rollback-plan": + return _run_calibration_rollback_plan(args) + parser.error("unknown command") return 2 diff --git a/tests/calibration/test_rollback_plan.py b/tests/calibration/test_rollback_plan.py new file mode 100644 index 00000000..c63f56e7 --- /dev/null +++ b/tests/calibration/test_rollback_plan.py @@ -0,0 +1,115 @@ +"""Tests for recalibration rollback plan.""" +from __future__ import annotations + +import json +import pytest +from pathlib import Path + +from openamp_foundry.calibration.rollback_plan import ( + ROLLBACK_TRIGGERS, + RollbackStep, + DEFAULT_ROLLBACK_STEPS, + build_rollback_plan, + write_rollback_plan_json, + write_rollback_plan_markdown, +) + + +class TestRollbackTriggers: + def test_at_least_5_triggers(self): + assert len(ROLLBACK_TRIGGERS) >= 5 + + def test_all_triggers_have_required_fields(self): + for trigger in ROLLBACK_TRIGGERS: + assert "id" in trigger + assert "name" in trigger + assert "description" in trigger + assert "threshold" in trigger + assert "severity" in trigger + + +class TestBuildRollbackPlan: + VALID_TRIGGERS = ["RT-01", "RT-02"] + + def test_valid_triggers_passes(self): + plan = build_rollback_plan("RBK-001", "v0.5.87", self.VALID_TRIGGERS) + assert plan.plan_id == "RBK-001" + assert plan.version == "v0.5.87" + + def test_unknown_trigger_raises_value_error(self): + with pytest.raises(ValueError, match="Unknown rollback trigger id"): + build_rollback_plan("RBK-002", "v0.5.87", ["RT-99"]) + + def test_triggered_by_stored_correctly(self): + plan = build_rollback_plan("RBK-003", "v0.5.87", ["RT-01", "RT-04"]) + assert "RT-01" in plan.triggered_by + assert "RT-04" in plan.triggered_by + assert len(plan.triggered_by) == 2 + + def test_steps_include_all_default_steps(self): + plan = build_rollback_plan("RBK-004", "v0.5.87", self.VALID_TRIGGERS) + default_actions = {s.action for s in DEFAULT_ROLLBACK_STEPS} + plan_actions = {s.action for s in plan.steps} + for action in default_actions: + assert action in plan_actions + + def test_extra_steps_appended_correctly(self): + extra = [ + RollbackStep(step_number=0, action="Notify stakeholders", responsible="human-reviewer", detail="Notify affected teams."), + ] + plan = build_rollback_plan("RBK-005", "v0.5.87", self.VALID_TRIGGERS, extra_steps=extra) + assert len(plan.steps) == len(DEFAULT_ROLLBACK_STEPS) + 1 + assert plan.steps[-1].action == "Notify stakeholders" + + def test_dry_lab_only_always_true(self): + plan = build_rollback_plan("RBK-006", "v0.5.87", self.VALID_TRIGGERS) + assert plan.dry_lab_only is True + + def test_plan_id_stored_correctly(self): + plan = build_rollback_plan("RBK-CUSTOM", "v0.5.87", self.VALID_TRIGGERS) + assert plan.plan_id == "RBK-CUSTOM" + + def test_version_stored_correctly(self): + plan = build_rollback_plan("RBK-007", "v0.5.88", self.VALID_TRIGGERS) + assert plan.version == "v0.5.88" + + def test_to_dict_has_required_keys(self): + plan = build_rollback_plan("RBK-008", "v0.5.87", self.VALID_TRIGGERS) + d = plan.to_dict() + for key in ("plan_id", "version", "triggered_by", "steps", "notes", "dry_lab_only"): + assert key in d + assert d["dry_lab_only"] is True + + def test_notes_stored_correctly(self): + plan = build_rollback_plan("RBK-009", "v0.5.87", self.VALID_TRIGGERS, notes="AUROC dropped 0.03.") + assert plan.notes == "AUROC dropped 0.03." + + def test_default_steps_have_correct_responsible_parties(self): + expected = {"pipeline", "human-reviewer", "both"} + for step in DEFAULT_ROLLBACK_STEPS: + assert step.responsible in expected + assert step.dry_lab_only is True + + +class TestWriters: + def test_write_rollback_plan_json(self, tmp_path: Path): + plan = build_rollback_plan("RBK-JSON", "v0.5.87", ["RT-01", "RT-04"]) + out = tmp_path / "rollback.json" + write_rollback_plan_json(plan, out) + assert out.exists() + data = json.loads(out.read_text()) + assert data["plan_id"] == "RBK-JSON" + assert data["dry_lab_only"] is True + assert len(data["steps"]) == len(DEFAULT_ROLLBACK_STEPS) + + def test_write_rollback_plan_markdown(self, tmp_path: Path): + plan = build_rollback_plan("RBK-MD", "v0.5.87", ["RT-01", "RT-04"]) + out = tmp_path / "rollback.md" + write_rollback_plan_markdown(plan, out) + assert out.exists() + content = out.read_text() + assert "# Recalibration Rollback Plan" in content + assert "RBK-MD" in content + assert "RT-01" in content + assert "RT-04" in content + assert "Caveats" in content diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index d5484aaf..3ece7aad 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 3063 tests as of Loop 87 (2026-07-09) -- Phase G G9 (calibration decision checklist) +# Baseline: 3078 tests as of Loop 88 (2026-07-09) -- Phase G G10 (recalibration rollback plan, Phase G complete) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3063 +BASELINE = 3078 TOLERANCE = 0.05 # allow 5% fluctuation