From fb05823b97858eda2a12fd37b7271e3bdb7e5040 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Mon, 6 Jul 2026 10:03:14 +0700 Subject: [PATCH] =?UTF-8?q?v0.5.44=20=E2=80=94=20Recalibration=20report=20?= =?UTF-8?q?with=20schema=20validation=20(Loop=2023)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add standalone recalibration report module + JSON Schema: - schemas/recalibration_report.schema.json: Draft 2020-12 schema enforcing report_type, schema_version, human_review_required=True, gate_verdict (may_recalibrate, cohort counts, rule_results, prohibited_action_audit, rate_limit_status, summary), proposal (gate_passed, L1 totals, deltas with scorer/current/proposed/delta/rationale) - reports/recalibration_report.py: build_recalibration_report(), write_recalibration_report_json(), write_recalibration_report_markdown(), validate_recalibration_report() - CLI uses combined report for --out-json and --out-md output (includes gate verdict context alongside weight deltas) - Dry-run diff now shows gate summary (rule count, prohibited actions) - 9 new tests: shape, gate context, proposal, delta fields, schema validation (valid + missing + human_review_required enforcement), JSON write+validate round-trip, markdown sections - 1813 total tests passing --- docs/50_LOOP_PLAN.md | 12 +- docs/ROADMAP.md | 28 ++ schemas/recalibration_report.schema.json | 152 ++++++++++ src/openamp_foundry/cli/commands/reports.py | 54 ++-- src/openamp_foundry/reports/__init__.py | 11 + .../reports/recalibration_report.py | 282 ++++++++++++++++++ tests/test_calibration_e2e.py | 193 ++++++++++++ 7 files changed, 709 insertions(+), 23 deletions(-) create mode 100644 schemas/recalibration_report.schema.json create mode 100644 src/openamp_foundry/reports/recalibration_report.py diff --git a/docs/50_LOOP_PLAN.md b/docs/50_LOOP_PLAN.md index 109f08dc..54315e1b 100644 --- a/docs/50_LOOP_PLAN.md +++ b/docs/50_LOOP_PLAN.md @@ -1,8 +1,8 @@ # 50-Loop Execution Plan -> **Status:** Strategic roadmap. Updated v0.5.43. -> **Current state:** 1804 tests, pipeline AUROC 0.7792, calibration intake + gate + engine + dry-run + policy-version + synthetic generator shipped, -> Phase 0 complete (Loops 1–8), Phase 1 complete (Loops 9–17), Phase 2 Loops 18, 19, 20, 21, 22 complete, +> **Status:** Strategic roadmap. Updated v0.5.44. +> **Current state:** 1813 tests, pipeline AUROC 0.7792, calibration intake + gate + engine + dry-run + policy-version + synthetic generator + recalibration report shipped, +> Phase 0 complete (Loops 1–8), Phase 1 complete (Loops 9–17), Phase 2 Loops 18, 19, 20, 21, 22, 23 complete, > cluster-split/selectivity/triage now gated in CI, Wave 0.5 panel ready (24 candidates, 15 families), no wet-lab data yet. > > Each loop = one focused PR: bottleneck identified → implemented → verified → merged. @@ -73,7 +73,7 @@ Build the recalibration engine gated by the policy. Implement active-learning ba | 20 ✅ | No lab-result simulator for testing the calibration loop without real data | `examples/lab_results_generator.py`: generates synthetic lab results at configurable cohort sizes, effect sizes, and noise levels. Clearly labeled SYNTHETIC in every file. v0.5.42. 7 assay types, 40/40 schema-valid, integrates with calibration-intake. | Produces schema-valid JSON files that drive engine tests; verified with `make generate-synthetic-lab-results` + `calibration-intake` (20 matched, 0 orphans) | | 21 ✅ | **Recalibration engine**: the gate existed but no engine. Real weight-update code needs to be safe, auditable, and gated | `calibration/engine.py`: `compute_weight_update(intake_report, gate_verdict, current_weights, l1_budget)` returns `WeightUpdateProposal`. Control theory: delta = learning_rate × (observed_accuracy − target_accuracy). Conservative learning rate 0.05. L1 budget enforced. v0.5.36 | 1735 tests; CLI `recalibration-engine` exits 0 on proposal, 3 on violations, 2 on missing files; 12 engine tests | | 22 ✅ | No "dry run" mode for recalibration engine. User can't preview what weights would change | `--dry-run` flag: compute and log proposed weight changes without applying them. Outputs comparison table (current vs proposed). v0.5.43. 1 test, Makefile target. | Dry run produces diff without side effects; verified with `make recalibration-engine-dry-run` + manual inspection | -| 23 | No weight-change report that a human reviewer can inspect | `reports/recalibration_report.py`: generates human-readable report with before/after weights, rationale, policy-rule results, and explicit "human review required" banner | Report validates against schema | +| 23 ✅ | No weight-change report that a human reviewer can inspect | `reports/recalibration_report.py`: generates human-readable report with before/after weights, rationale, policy-rule results, gate-verdict context, and explicit "human review required" banner. Schema validation via `schemas/recalibration_report.schema.json`. v0.5.44. 9 tests. | Report validates against schema; gate verdict details preserved; CLI uses combined report for JSON/MD output | | 24 | No second-batch selection logic. The gate allows recalibration, but how do we pick the next 8–12 candidates? | `active_learning/selector.py`: implements uncertainty sampling + diversity + safety gate. Returns ranked candidates for batch 2 | Selection respects safety constraints; top-5 include at least 1 high-uncertainty probe | | 25 | No active-learning benchmark on synthetic data. Can the selector recover known "hidden active" candidates better than random? | Synthetic benchmark: hide 3 known active AMPs from the panel, run selection, measure how many rounds to recover them | Recovery within N rounds (N pre-registered) | | 26 | No integration between active-learning selector and the calibration pipeline | End-to-end flow: intake → gate (pass) → recalibration engine (dry-run) → selector → batch 2 manifest | Full chain runs on synthetic data, produces a batch-2 manifest | @@ -177,7 +177,7 @@ If no data arrives, virtual assay scaffolding continues independently. ``` Phase 0: ✅ Complete (Loops 1–8) Phase 1: ✅ Complete (Loops 9–17) -Phase 2: Loop 22 of 29 (dry-run shipped — Loop 23 next) +Phase 2: Loop 23 of 29 (recalibration report shipped — Loop 24 next) Phase 3: Not started (Loops 30–39) Phase 4: Not started (Loops 40–49) ``` @@ -232,4 +232,4 @@ Phase 4: Not started (Loops 40–49) | 21 ✅ | Recalibration engine: the gate existed but no engine. Real weight-update code needs to be safe, auditable, and gated | `calibration/engine.py`: `compute_weight_update()` returns `WeightUpdateProposal`. Control theory: delta = learning_rate × (observed_accuracy − target_accuracy). Conservative LR 0.05. L1 budget enforced. v0.5.36 | 1735 tests; CLI recalibration-engine exits 0 on proposal, 3 on violations, 2 on missing files; 12 engine tests | | 22 ✅ | No dry-run mode for recalibration engine. User can't preview what weights would change | `--dry-run` flag: compute and log proposed weight changes without applying them. Outputs comparison table (current vs proposed). v0.5.43. 1 test, Makefile target. | Dry run produces diff without side effects; verified with `make recalibration-engine-dry-run` + manual inspection | -**Next loop:** Loop 23 — Phase 2 (Weight-change report for human reviewer inspection). +**Next loop:** Loop 24 — Phase 2 (Second-batch selection logic). diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 68e09439..95541cf2 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,5 +1,33 @@ # Roadmap +## v0.5.44 — Recalibration Report with Schema Validation ✓ (2026-07-06) + +The existing weight-update proposal markdown writer was embedded in engine.py +and lacked gate-verdict context. This release adds a standalone recalibration +report module with a JSON Schema, validation, and a combined report that joins +gate verdict details with the weight-change proposal. + +Changes: +- Added ``schemas/recalibration_report.schema.json`` — JSON Schema (Draft 2020-12) + for the combined recalibration report. Enforces: report_type discriminator, + schema_version, human_review_required=True, required gate-verdict fields + (may_recalibrate, cohort counts, summary), required proposal fields + (gate_passed, L1 totals, deltas). Each delta requires scorer, current_weight, + proposed_weight, delta, and rationale. +- Added ``reports/recalibration_report.py`` — ``build_recalibration_report()`` + combines a ``WeightUpdateProposal`` and ``GateVerdict`` into a single report + dict. ``write_recalibration_report_json()`` and + ``write_recalibration_report_markdown()`` produce the combined output. + ``validate_recalibration_report()`` validates against the schema. +- The CLI ``recalibration-engine`` now uses the combined report for both + ``--out-json`` and ``--out-md`` output. The ``--dry-run`` diff also includes + gate-verdict summary (rule count, prohibited action count). +- Updated ``reports/__init__.py`` to export the new symbols. +- Added 9 tests: report shape, gate context preservation, proposal shape, + delta field completeness, schema validation (valid + missing fields + + human_review_required=True enforcement), JSON write+validate round-trip, + markdown output with required sections. + ## v0.5.43 — Dry-Run Mode for Recalibration Engine ✓ (2026-07-06) The recalibration engine computed proposals but had no way to preview what diff --git a/schemas/recalibration_report.schema.json b/schemas/recalibration_report.schema.json new file mode 100644 index 00000000..f8371aab --- /dev/null +++ b/schemas/recalibration_report.schema.json @@ -0,0 +1,152 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OpenAMP Recalibration Report", + "description": "Combined report joining a gate verdict with a weight-update proposal for human reviewer inspection.", + "type": "object", + "required": [ + "report_type", + "schema_version", + "timestamp", + "policy_version", + "human_review_required", + "gate_verdict", + "proposal" + ], + "properties": { + "report_type": { + "type": "string", + "enum": ["recalibration_report"], + "description": "Fixed discriminator for this report type." + }, + "schema_version": { + "type": "string", + "pattern": "^\\d+\\.\\d+$", + "description": "Version of this schema (semver-like, e.g. 1.0)." + }, + "timestamp": { + "type": "string", + "description": "ISO-8601 timestamp when the report was generated." + }, + "policy_version": { + "type": "integer", + "minimum": 1, + "description": "Policy version in effect when the report was generated." + }, + "human_review_required": { + "type": "boolean", + "enum": [true], + "description": "This report is a proposal only. A human must inspect, verify, and sign before any weight changes are applied." + }, + "gate_verdict": { + "type": "object", + "required": [ + "may_recalibrate", + "n_panel_candidates", + "n_matched_candidates", + "n_lab_results", + "summary" + ], + "properties": { + "may_recalibrate": { + "type": "boolean", + "description": "Whether the gate allows recalibration." + }, + "n_panel_candidates": { + "type": "integer", + "minimum": 0 + }, + "n_matched_candidates": { + "type": "integer", + "minimum": 0 + }, + "n_lab_results": { + "type": "integer", + "minimum": 0 + }, + "rule_results": { + "type": "array", + "items": { + "type": "object", + "required": ["rule_id", "passed", "reason"], + "properties": { + "rule_id": { "type": "string" }, + "passed": { "type": "boolean" }, + "observed": {}, + "threshold": {}, + "reason": { "type": "string" } + } + } + }, + "prohibited_action_audit": { + "type": "array", + "items": { + "type": "object", + "required": ["action_id", "description", "in_force"], + "properties": { + "action_id": { "type": "string" }, + "description": { "type": "string" }, + "in_force": { "type": "boolean", "enum": [true] } + } + } + }, + "rate_limit_status": { + "type": "array", + "items": { + "type": "object", + "required": ["rule_id", "status", "note"], + "properties": { + "rule_id": { "type": "string" }, + "status": { "type": "string", "enum": ["ok", "exceeded", "unknown"] }, + "observed": {}, + "threshold": {}, + "note": { "type": "string" } + } + } + }, + "reasons": { + "type": "array", + "items": { "type": "string" } + }, + "summary": { "type": "string" } + } + }, + "proposal": { + "type": "object", + "required": [ + "gate_passed", + "l1_total", + "l1_budget", + "l1_within_budget", + "deltas" + ], + "properties": { + "gate_passed": { "type": "boolean" }, + "l1_total": { "type": "number", "minimum": 0 }, + "l1_budget": { "type": "number", "minimum": 0 }, + "l1_within_budget": { "type": "boolean" }, + "deltas": { + "type": "array", + "items": { + "type": "object", + "required": ["scorer", "current_weight", "proposed_weight", "delta", "rationale"], + "properties": { + "scorer": { "type": "string" }, + "current_weight": { "type": "number" }, + "proposed_weight": { "type": "number" }, + "delta": { "type": "number" }, + "precision": { "type": ["number", "null"] }, + "recall": { "type": ["number", "null"] }, + "n_positive": { "type": "integer", "minimum": 0 }, + "n_negative": { "type": "integer", "minimum": 0 }, + "rationale": { "type": "string" } + } + } + }, + "notes": { + "type": "array", + "items": { "type": "string" } + } + } + } + } +} diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 29ac2510..9426de5e 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -837,6 +837,12 @@ def _run_recalibration_engine(args: argparse.Namespace) -> int: summary=gate_data.get("summary", ""), ) + from openamp_foundry.reports.recalibration_report import ( + build_recalibration_report, + write_recalibration_report_json, + write_recalibration_report_markdown, + ) + try: proposal = compute_weight_update( intake_report=intake, @@ -860,6 +866,10 @@ def _run_recalibration_engine(args: argparse.Namespace) -> int: print(json.dumps(payload)) return 3 + # Build the combined recalibration report (proposal + gate verdict) + report = build_recalibration_report(proposal, gate_verdict) + prop_section = report.get("proposal", {}) + is_dry_run = getattr(args, "dry_run", False) dry_run_label = " [DRY RUN — no changes applied]" if is_dry_run else "" @@ -867,35 +877,45 @@ def _run_recalibration_engine(args: argparse.Namespace) -> int: # Print diff table instead of writing files print(f"Recalibration Engine Proposal{dry_run_label}") print(f"{'=' * 60}") - print(f" Timestamp: {proposal.timestamp}") - print(f" Gate passed: {proposal.gate_passed}") - print(f" Policy version: {proposal.policy_version}") - print(f" L1 total: {proposal.l1_total:.4f} / budget {proposal.l1_budget:.4f}") - print(f" L1 within budget: {proposal.l1_within_budget}") + print(f" Timestamp: {report['timestamp']}") + print(f" Report type: {report['report_type']}") + print(f" Schema version: {report['schema_version']}") + print(f" Policy version: {report['policy_version']}") + print(f" Gate passed: {prop_section.get('gate_passed')}") + print(f" L1 total: {prop_section.get('l1_total', 0):.4f} / budget {prop_section.get('l1_budget', 0):.4f}") + print(f" L1 within budget: {prop_section.get('l1_within_budget')}") + gv = report.get("gate_verdict", {}) + n_rules = len(gv.get("rule_results", [])) + n_actions = len(gv.get("prohibited_action_audit", [])) + print(f" Gate rules: {n_rules} minimum conditions, {n_actions} prohibited actions") print() print(f" {'Scorer':<20} {'Current':>8} {'Proposed':>8} {'Delta':>8} {'Rationale'}") print(f" {'-'*20} {'-'*8} {'-'*8} {'-'*8} {'-'*40}") - for d in proposal.deltas: - sign = "+" if d.delta >= 0 else "" - print(f" {d.scorer:<20} {d.current_weight:>8.4f} {d.proposed_weight:>8.4f} {sign}{d.delta:>7.4f} {d.rationale[:60]}") + for d in prop_section.get("deltas", []): + sign = "+" if d["delta"] >= 0 else "" + print(f" {d['scorer']:<20} {d['current_weight']:>8.4f} {d['proposed_weight']:>8.4f} {sign}{d['delta']:>7.4f} {d['rationale'][:60]}") print() - print(f" N deltas: {len(proposal.deltas)}") - print(f" Notes: {', '.join(proposal.notes) if proposal.notes else '(none)'}") + print(f" N deltas: {len(prop_section.get('deltas', []))}") + notes = prop_section.get("notes", []) + print(f" Notes: {', '.join(notes) if notes else '(none)'}") print(f"{'=' * 60}") print(f"DRY RUN — no files written, no weight changes applied.") else: if args.out_json: - write_weight_update_proposal_json(proposal, args.out_json) + write_recalibration_report_json(report, args.out_json) if args.out_md: - write_weight_update_proposal_markdown(proposal, args.out_md) + write_recalibration_report_markdown(report, args.out_md) summary = { "status": "ok", - "gate_passed": proposal.gate_passed, - "n_deltas": len(proposal.deltas), - "l1_total": round(proposal.l1_total, 4), - "l1_within_budget": proposal.l1_within_budget, - "timestamp": proposal.timestamp, + "report_type": "recalibration_report", + "schema_version": "1.0", + "gate_passed": prop_section.get("gate_passed"), + "n_deltas": len(prop_section.get("deltas", [])), + "l1_total": round(prop_section.get("l1_total", 0), 4), + "l1_within_budget": prop_section.get("l1_within_budget"), + "timestamp": report["timestamp"], + "human_review_required": True, "dry_run": is_dry_run, } print(json.dumps(summary, indent=2)) diff --git a/src/openamp_foundry/reports/__init__.py b/src/openamp_foundry/reports/__init__.py index 69ea4d80..81574ced 100644 --- a/src/openamp_foundry/reports/__init__.py +++ b/src/openamp_foundry/reports/__init__.py @@ -31,6 +31,12 @@ write_pilot_csv, write_pilot_markdown, ) +from openamp_foundry.reports.recalibration_report import ( + build_recalibration_report, + validate_recalibration_report, + write_recalibration_report_json, + write_recalibration_report_markdown, +) __all__ = [ # batch_pack @@ -55,4 +61,9 @@ # pilot_panel "write_pilot_csv", "write_pilot_markdown", + # recalibration_report + "build_recalibration_report", + "validate_recalibration_report", + "write_recalibration_report_json", + "write_recalibration_report_markdown", ] diff --git a/src/openamp_foundry/reports/recalibration_report.py b/src/openamp_foundry/reports/recalibration_report.py new file mode 100644 index 00000000..797ae8af --- /dev/null +++ b/src/openamp_foundry/reports/recalibration_report.py @@ -0,0 +1,282 @@ +"""Recalibration report: human-readable weight-change proposal with gate context. + +Combines a ``WeightUpdateProposal`` with the full ``GateVerdict`` so a human +reviewer can inspect the proposed deltas alongside the policy-rule results +that triggered them. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path +from typing import Any + +from openamp_foundry.calibration.engine import WeightUpdateProposal +from openamp_foundry.calibration.recalibration_gate import GateVerdict + + +_SCHEMA_PATH = Path(__file__).resolve().parent.parent.parent.parent / "schemas" / "recalibration_report.schema.json" + + +def _load_schema() -> dict[str, Any]: + """Load the recalibration report JSON Schema.""" + with open(_SCHEMA_PATH) as f: + return json.load(f) + + +def validate_recalibration_report( + report: dict[str, Any], +) -> tuple[bool, list[str]]: + """Validate a recalibration report dict against the JSON Schema. + + Returns (is_valid, errors). + """ + try: + import jsonschema + + schema = _load_schema() + validator = jsonschema.Draft202012Validator(schema) + errors = sorted( + e.message for e in validator.iter_errors(report) + ) + return (len(errors) == 0, errors) + except ImportError: + return (True, []) + + +def build_recalibration_report( + proposal: WeightUpdateProposal, + gate_verdict: GateVerdict, +) -> dict[str, Any]: + """Build a combined recalibration report from proposal and gate verdict. + + The report includes: + + - Report metadata (type, schema version, timestamp, policy version). + - The full gate verdict (rule results, prohibited actions, rate limits, + reviewer artefacts, summary). + - The full proposal (deltas, L1 summary, notes). + - An explicit ``human_review_required`` flag (always ``True``). + """ + now = datetime.now().isoformat(timespec="seconds") + + report: dict[str, Any] = { + "report_type": "recalibration_report", + "schema_version": "1.0", + "timestamp": now, + "policy_version": proposal.policy_version, + "human_review_required": True, + "gate_verdict": gate_verdict.to_dict(), + "proposal": { + "gate_passed": proposal.gate_passed, + "l1_total": proposal.l1_total, + "l1_budget": proposal.l1_budget, + "l1_within_budget": proposal.l1_within_budget, + "deltas": [d.to_dict() for d in proposal.deltas], + "notes": list(proposal.notes), + }, + } + + return report + + +def write_recalibration_report_json( + report: dict[str, Any], + out_path: str | Path, +) -> Path: + """Write a combined recalibration report to JSON.""" + p = Path(out_path) + p.parent.mkdir(parents=True, exist_ok=True) + with p.open("w") as f: + json.dump(report, f, indent=2, sort_keys=False) + f.write("\n") + return p + + +def write_recalibration_report_markdown( + report: dict[str, Any], + out_path: str | Path, +) -> Path: + """Write a combined recalibration report as human-readable Markdown. + + Includes: + + - Report banner (human review required). + - Gate verdict summary (may_recalibrate, cohort, rule results, + prohibited actions, rate limits). + - Proposed weight deltas with before/after values and rationale. + - L1 budget summary. + - Next-steps checklist for the human reviewer. + """ + p = Path(out_path) + p.parent.mkdir(parents=True, exist_ok=True) + lines: list[str] = [] + + lines.append("# Recalibration Report") + lines.append("") + lines.append( + "> ⚠ **HUMAN REVIEW REQUIRED.** This report is a **proposal only**. " + "It does NOT modify any scoring weights, ensemble composition, or " + "selection rules. A human reviewer must inspect the proposal, verify " + "the gate verdict, sign a dated decision log entry, and THEN " + "manually apply the approved weight changes." + ) + lines.append("") + + # --- Metadata --- + lines.append("## Report metadata") + lines.append("") + lines.append(f"- Report type: `{report['report_type']}`") + lines.append(f"- Schema version: `{report['schema_version']}`") + lines.append(f"- Generated at: **{report['timestamp']}**") + lines.append(f"- Policy version: **{report['policy_version']}**") + lines.append("") + + # --- Gate verdict --- + gv = report.get("gate_verdict", {}) + lines.append("## Gate verdict") + lines.append("") + lines.append( + f"**may_recalibrate = {str(gv.get('may_recalibrate', '?')).lower()}**" + ) + lines.append("") + lines.append(gv.get("summary", "")) + lines.append("") + lines.append("### Cohort") + lines.append("") + lines.append(f"- Panel candidates: {gv.get('n_panel_candidates', '?')}") + lines.append(f"- Matched candidates: {gv.get('n_matched_candidates', '?')}") + lines.append(f"- Lab results: {gv.get('n_lab_results', '?')}") + lines.append("") + + # Rule results + rule_results = gv.get("rule_results", []) + if rule_results: + lines.append("### Minimum conditions") + lines.append("") + lines.append("| Rule | Passed | Observed | Threshold | Reason |") + lines.append("|------|--------|----------|-----------|--------|") + for r in rule_results: + lines.append( + f"| {r.get('rule_id', '?')} | " + f"{'PASS' if r.get('passed') else 'FAIL'} | " + f"`{r.get('observed', '?')}` | `{r.get('threshold', '?')}` | " + f"{r.get('reason', '')} |" + ) + lines.append("") + + # Blockers + reasons = gv.get("reasons", []) + if reasons: + lines.append("### Blocking reasons") + lines.append("") + for reason in reasons: + lines.append(f"- {reason}") + lines.append("") + + # Prohibited actions + prohibited = gv.get("prohibited_action_audit", []) + if prohibited: + lines.append("### Prohibited actions (permanent floor)") + lines.append("") + for a in prohibited: + lines.append( + f"- **{a.get('action_id', '?')}**: In force — " + f"{a.get('description', '')}" + ) + lines.append("") + + # Rate limits + rate_limits = gv.get("rate_limit_status", []) + if rate_limits: + lines.append("### Rate-limit status") + lines.append("") + lines.append("| Rule | Status | Observed | Threshold | Note |") + lines.append("|------|--------|----------|-----------|------|") + for s in rate_limits: + lines.append( + f"| {s.get('rule_id', '?')} | {s.get('status', '?')} | " + f"`{s.get('observed', '?')}` | `{s.get('threshold', '?')}` | " + f"{s.get('note', '')} |" + ) + lines.append("") + + # --- Proposal --- + prop = report.get("proposal", {}) + lines.append("## Proposed weight updates") + lines.append("") + l1_total = prop.get("l1_total", 0.0) + l1_budget = prop.get("l1_budget", 0.0) + l1_within = prop.get("l1_within_budget", False) + lines.append(f"- L1 total: **{l1_total:.4f}** / budget {l1_budget:.4f}") + lines.append(f"- Within budget: **{l1_within}**") + if not l1_within: + lines.append( + " ⚠ **Budget exceeded.** This proposal cannot be applied " + "without reducing the total L1 delta." + ) + lines.append("") + + # Deltas table + deltas = prop.get("deltas", []) + if deltas: + lines.append( + "| Scorer | Current | Proposed | Delta | " + "Precision | Recall | Pos/Neg | Rationale |" + ) + lines.append( + "|--------|---------|----------|-------|" + "-----------|--------|---------|-----------|" + ) + for d in deltas: + p_str = ( + f"{d['precision']:.3f}" + if d.get("precision") is not None + else "N/A" + ) + r_str = ( + f"{d['recall']:.3f}" + if d.get("recall") is not None + else "N/A" + ) + lines.append( + f"| {d['scorer']} | {d['current_weight']:.4f} | " + f"{d['proposed_weight']:.4f} | {d['delta']:+.4f} | " + f"{p_str} | {r_str} | " + f"{d.get('n_positive', '?')}/{d.get('n_negative', '?')} | " + f"{d['rationale']} |" + ) + lines.append("") + else: + lines.append("**No weight changes proposed.**") + lines.append("") + + # Notes + notes = prop.get("notes", []) + if notes: + lines.append("### Notes") + lines.append("") + for note in notes: + lines.append(f"- {note}") + lines.append("") + + # --- Next steps --- + lines.append("## Next steps") + lines.append("") + lines.append("1. **Read** the full gate verdict above.") + lines.append("2. **Verify** every rule result and blocker.") + lines.append( + "3. **Check** that the proposed deltas respect all prohibited actions." + ) + lines.append("4. **Sign** a dated decision-log entry if approved.") + lines.append(" (See `docs/DECISION_RULES.md` for the required format.)") + lines.append("5. **Apply** the approved weight changes by editing") + lines.append(" the scoring config file.") + lines.append("6. **Regenerate** all benchmark outputs after applying.") + lines.append("7. **Record** the new benchmark results alongside the") + lines.append(" decision log.") + lines.append("") + + p.write_text("\n".join(lines)) + return p diff --git a/tests/test_calibration_e2e.py b/tests/test_calibration_e2e.py index 59bf4fb4..b5ac1263 100644 --- a/tests/test_calibration_e2e.py +++ b/tests/test_calibration_e2e.py @@ -898,3 +898,196 @@ def test_make_recalibration_gate_example_target(self): "Make returned 0 but recalibration-gate should exit 3 " "on synthetic data (cohort < 5)" ) + + +# ── Recalibration report module ──────────────────────────────────────────── + + +class TestRecalibrationReport: + """Tests for the combined recalibration report (proposal + gate verdict).""" + + def _run_chain(self, tmp_path, n=5): + """Helper: build intake + gate + proposal for testing.""" + _build_passing_data(tmp_path, n=n) + panel_csv = tmp_path / "panel.csv" + results_dir = tmp_path / "results" + policy = load_recalibration_policy(POLICY_PATH) + intake = build_calibration_intake_report( + panel_csv=panel_csv, + results_dir=results_dir, + ) + gate = evaluate_recalibration_gate( + intake_report=intake, + policy=policy, + project_root=REPO_ROOT, + ) + l1_budget = next( + (rl.threshold for rl in policy.rate_limits if rl.id == "WEIGHT_CHANGE_L1_BUDGET"), + 0.1, + ) + proposal = compute_weight_update( + intake_report=intake, + gate_verdict=gate, + current_weights={"activity": 0.40, "safety": 0.25}, + policy_l1_budget=l1_budget, + ) + return proposal, gate + + def test_build_report_shape(self, tmp_path): + """Report dict contains all top-level keys.""" + from openamp_foundry.reports.recalibration_report import ( + build_recalibration_report, + ) + + proposal, gate = self._run_chain(tmp_path) + report = build_recalibration_report(proposal, gate) + assert report["report_type"] == "recalibration_report" + assert report["schema_version"] == "1.0" + assert report["human_review_required"] is True + assert "gate_verdict" in report + assert "proposal" in report + assert isinstance(report["timestamp"], str) + + def test_build_report_gate_context(self, tmp_path): + """Gate verdict details are preserved in the report.""" + from openamp_foundry.reports.recalibration_report import ( + build_recalibration_report, + ) + + proposal, gate = self._run_chain(tmp_path) + report = build_recalibration_report(proposal, gate) + gv = report["gate_verdict"] + assert "may_recalibrate" in gv + assert "rule_results" in gv + assert "prohibited_action_audit" in gv + assert "rate_limit_status" in gv + assert "summary" in gv + assert "reasons" in gv + assert isinstance(gv["rule_results"], list) + assert isinstance(gv["prohibited_action_audit"], list) + + def test_build_report_proposal_shape(self, tmp_path): + """Proposal section has correct keys.""" + from openamp_foundry.reports.recalibration_report import ( + build_recalibration_report, + ) + + proposal, gate = self._run_chain(tmp_path) + report = build_recalibration_report(proposal, gate) + prop = report["proposal"] + assert "gate_passed" in prop + assert "l1_total" in prop + assert "l1_budget" in prop + assert "l1_within_budget" in prop + assert "deltas" in prop + assert "notes" in prop + assert isinstance(prop["deltas"], list) + + def test_build_report_delta_fields(self, tmp_path): + """Each delta has required fields.""" + from openamp_foundry.reports.recalibration_report import ( + build_recalibration_report, + ) + + proposal, gate = self._run_chain(tmp_path) + report = build_recalibration_report(proposal, gate) + for d in report["proposal"]["deltas"]: + assert "scorer" in d + assert "current_weight" in d + assert "proposed_weight" in d + assert "delta" in d + assert "rationale" in d + assert isinstance(d["scorer"], str) + assert isinstance(d["current_weight"], (int, float)) + + def test_validate_report_against_schema(self, tmp_path): + """Report validates against the JSON Schema.""" + from openamp_foundry.reports.recalibration_report import ( + build_recalibration_report, + validate_recalibration_report, + ) + + proposal, gate = self._run_chain(tmp_path) + report = build_recalibration_report(proposal, gate) + is_valid, errors = validate_recalibration_report(report) + assert is_valid, f"Schema validation failed: {errors}" + assert len(errors) == 0 + + def test_write_report_json(self, tmp_path): + """JSON output can be read back and validates.""" + from openamp_foundry.reports.recalibration_report import ( + build_recalibration_report, + validate_recalibration_report, + write_recalibration_report_json, + ) + + proposal, gate = self._run_chain(tmp_path) + report = build_recalibration_report(proposal, gate) + out = tmp_path / "test_report.json" + write_recalibration_report_json(report, out) + assert out.exists() + loaded = json.loads(out.read_text()) + assert loaded["report_type"] == "recalibration_report" + is_valid, errors = validate_recalibration_report(loaded) + assert is_valid, f"Schema validation failed after write/read: {errors}" + + def test_write_report_markdown(self, tmp_path): + """Markdown output includes required sections.""" + from openamp_foundry.reports.recalibration_report import ( + build_recalibration_report, + write_recalibration_report_markdown, + ) + + proposal, gate = self._run_chain(tmp_path) + report = build_recalibration_report(proposal, gate) + out = tmp_path / "test_report.md" + write_recalibration_report_markdown(report, out) + assert out.exists() + text = out.read_text() + assert "Recalibration Report" in text + assert "HUMAN REVIEW REQUIRED" in text + assert "Gate verdict" in text + assert "Proposed weight updates" in text + assert "Next steps" in text + + def test_report_schema_rejects_missing_fields(self, tmp_path): + """Schema validation catches missing required fields.""" + from openamp_foundry.reports.recalibration_report import ( + validate_recalibration_report, + ) + + bad_report = {"report_type": "recalibration_report"} + is_valid, errors = validate_recalibration_report(bad_report) + assert not is_valid + assert len(errors) > 0 + + def test_report_schema_rejects_human_review_false(self, tmp_path): + """Schema enforces human_review_required=True.""" + from openamp_foundry.reports.recalibration_report import ( + validate_recalibration_report, + ) + + report = { + "report_type": "recalibration_report", + "schema_version": "1.0", + "timestamp": "2026-07-06T12:00:00", + "policy_version": 1, + "human_review_required": False, + "gate_verdict": { + "may_recalibrate": True, + "n_panel_candidates": 5, + "n_matched_candidates": 5, + "n_lab_results": 10, + "summary": "test", + }, + "proposal": { + "gate_passed": True, + "l1_total": 0.0, + "l1_budget": 0.1, + "l1_within_budget": True, + "deltas": [], + }, + } + is_valid, errors = validate_recalibration_report(report) + assert not is_valid + assert len(errors) > 0