Phase G G3 — calibration pipeline consistency audit

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard
+.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example
 
 PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
 PYTEST  := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
@@ -75,7 +75,9 @@ help:
 	@echo "  make validate-policy-version    Validate proposed policy version against predecessor"
 	@echo "  make bump-policy-version        Bump policy version with decision-log guard (requires --human-reviewer)"
 	@echo "  make generate-synthetic-lab-results  Generate synthetic lab results for calibration testing"
-	@echo "  make test               Run full test suite (1803 passing tests, >=80% coverage)"
+	@echo "  make calibration-audit-example  Run calibration pipeline consistency audit on synthetic example"
+	@echo "  make calibration-audit          Run calibration pipeline consistency audit (INTAKE=[path] GATE=[path] ...)"
+	@echo "  make test               Run full test suite (2937 passing tests, >=80% coverage)"
 	@echo "  make coverage           Test suite with per-module coverage report"
 	@echo "  make lint               Ruff lint check on src/ tests/ scripts/"
 	@echo "  make typecheck          mypy type check on src/"
@@ -479,6 +481,32 @@ calibration-loop:
 		--out-dir outputs/calibration_loop \
 		--seed 42 --n-batch-2 10
 
+calibration-audit-example: lab-result-intake-example
+	-PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli recalibration-gate \
+		--intake-report outputs/calibration_intake_example.json \
+		--intake-report-date 2026-07-04 \
+		--out-json outputs/recalibration_gate_example.json \
+		--out-md outputs/recalibration_gate_example.md 2>/dev/null
+	-PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli calibration-audit \
+		--intake-report outputs/calibration_intake_example.json \
+		--gate-verdict outputs/recalibration_gate_example.json \
+		--out-json outputs/calibration_audit_example.json \
+		--out-md outputs/calibration_audit_example.md
+
+calibration-audit:
+	@if [ -z "$(INTAKE)" ] && [ -z "$(GATE)" ] && [ -z "$(ENGINE)" ] && [ -z "$(REPORT)" ]; then \
+		echo "Usage: make calibration-audit [INTAKE=<intake.json>] [GATE=<gate.json>] [ENGINE=<engine.json>] [REPORT=<report.json>] [OUT_JSON=<path>] [OUT_MD=<path>]"; \
+		echo "At least one artifact path is required."; \
+		exit 1; \
+	fi
+	PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli calibration-audit \
+		$$(if [ -n "$(INTAKE)" ]; then echo --intake-report "$(INTAKE)"; fi) \
+		$$(if [ -n "$(GATE)" ]; then echo --gate-verdict "$(GATE)"; fi) \
+		$$(if [ -n "$(ENGINE)" ]; then echo --engine-proposal "$(ENGINE)"; fi) \
+		$$(if [ -n "$(REPORT)" ]; then echo --recalibration-report "$(REPORT)"; fi) \
+		$$(if [ -n "$(OUT_JSON)" ]; then echo --out-json "$(OUT_JSON)"; fi) \
+		$$(if [ -n "$(OUT_MD)" ]; then echo --out-md "$(OUT_MD)"; fi)
+
 generate-synthetic-lab-results:
 	PYTHONPATH=src $(PYTHON) examples/lab_results_generator.py \
 		--panel-csv outputs/pilot_panel.csv \

docs/evidence/METRICS_CURRENT.md

@@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
 > with this file, this file wins. Updated whenever benchmark/benchmark config changes.
 >
-> **Last updated:** 2026-07-09 (Phase F F10 — negative-result archive completeness checker — v0.5.80)
+> **Last updated:** 2026-07-09 (Phase G G3 — calibration pipeline consistency audit — v0.5.81)
+> **New in v0.5.81:** Calibration pipeline consistency audit (G3) — CLI (`openamp-foundry calibration-audit`) that checks consistency across calibration pipeline artifacts (intake report, gate verdict, engine proposal, recalibration report). Checks intake↔gate count matching, engine↔gate verdict agreement, engine L1 budget compliance, report↔gate verdict consistency, report↔engine proposal consistency, timestamp sanity, and cohort-metrics warnings. Schema (`schemas/calibration_audit.schema.json`). `make calibration-audit` and `make calibration-audit-example` targets. 18 tests. 2937 total.
 > **New in v0.5.80:** Negative-result archive completeness checker (F10) — CLI (`scripts/check_negative_archive_completeness.py`) that reads a JSON archive of negative-result entries and checks each entry against completeness criteria: all required fields present, no duplicate candidate_ids, each entry has at least one content field (assay_result, score_safety, reviewer_notes, or reason_detail), date format valid YYYY-MM-DD, and optional intake_report_id references are well-formed. Outputs a structured completeness report as JSON and Markdown with summary, per-check results, and per-entry pass/fail. Schema (`schemas/negative_result_archive_completeness.schema.json`). Example file (`examples/negative_result_archive_example.json`) with 4 toy entries. 35 tests. 2919 total.
 >
 > **New in v0.5.79:** Negative-result dashboard (F9) — CLI (`scripts/negative_result_dashboard.py`) that reads a collection of negative-result entries from a JSON file and produces a structured dashboard with summary statistics (by category, by pipeline version), score distributions (activity, safety, novelty, ensemble), per-category cross-analysis, and pipeline insights (most common failure category, recalibration opportunities). Outputs JSON + Markdown. Example file (`examples/negative_result_dashboard_example.json`) with 15 toy entries across all 6 failure categories. Schema (`schemas/negative_result_dashboard.schema.json`). `make negative-result-dashboard` target. 33 tests. 2883 total.
@@ -72,7 +73,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test).
 > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination.
 > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison.
-> **Pipeline version:** v0.5.79 (F9)
+> **Pipeline version:** v0.5.81 (G3)
 > **Branch:** main
 
 ---

docs/research/ROADMAP.md

@@ -1,5 +1,55 @@
 # Roadmap
 
+## v0.5.81 — Loop 81: Phase G G3 — Calibration Pipeline Consistency Audit ✓ (2026-07-09)
+
+CLI (`openamp-foundry calibration-audit`) that checks consistency across the
+calibration pipeline artifacts — intake report, gate verdict, engine proposal,
+and combined recalibration report. Ensures a human reviewer inspecting the
+calibration pipeline output can verify that all stages agree on candidate
+counts, gate verdicts, weight proposals, and timestamps.
+
+Changes:
+- `src/openamp_foundry/calibration/audit.py` (G3) — `run_calibration_audit()`
+  accepts file paths or pre-loaded dicts for any combination of the four
+  artifacts. Runs 12 consistency checks: artifact path existence, intake↔gate
+  count matching, engine↔gate verdict agreement, engine L1 budget compliance,
+  engine intake-link match, report↔gate verdict match, report↔engine proposal
+  match, timestamp sanity, and intake cohort-metrics warnings. Each check has
+  check_id, description, pass/fail, observed, expected, and severity (error/
+  warning/info). Returns structured dict with overall_pass, checks array, and
+  summary.
+- `schemas/calibration_audit.schema.json` (G3) — JSON Schema Draft 2020-12 for
+  the calibration audit report. Validates report_type, schema_version, timestamp,
+  artifacts_checked, overall_pass, checks array, and summary.
+- `src/openamp_foundry/cli/commands/reports.py` — Added `_run_calibration_audit`
+  CLI handler with `--intake-report`, `--gate-verdict`, `--engine-proposal`,
+  `--recalibration-report`, `--out-json`, `--out-md` flags.
+- `src/openamp_foundry/cli/main.py` — Registered `calibration-audit` subcommand
+  with all argument flags and dispatch to `_run_calibration_audit`.
+- `Makefile` — Added `calibration-audit-example` and `calibration-audit` targets.
+  Example target runs on synthetic intake + gate outputs.
+- `tests/calibration/test_calibration_audit.py` — 18 tests covering: no
+  artifact edge case, single artifact, intake↔gate count match/mismatch,
+  engine↔gate verdict match/mismatch, L1 budget within/exceeds, report↔gate
+  match, report↔engine match, future timestamp detection, cohort-metrics
+  warning, JSON schema conformance, Markdown output, synthetic example
+  consistency, synthetic path existence, engine without gate_passed, and
+  nonexistent path handling.
+- `docs/evidence/METRICS_CURRENT.md` — v0.5.81 G3 changelog. Test count: 2937.
+- `tests/test_test_count_regression.py` — baseline updated to 2937.
+
+Honest boundaries:
+- This audit checks **consistency** between pipeline artifacts, not biological
+  validity. A passing audit means the pipeline stages agreed with each other,
+  not that calibration decisions are correct.
+- All tests use synthetic dict fixtures, not real wet-lab data.
+- Timestamp checks detect future timestamps but do not validate chronological
+  ordering between artifacts (e.g., gate must precede engine). This is a known
+  limitation for a future iteration.
+- The audit does not validate artifact schema conformance (each artifact's
+  schema validity is tested separately). It focuses on cross-artifact
+  consistency.
+
 ## v0.5.80 — Loop 80: Phase F F10 — Negative-Result Archive Completeness Checker ✓ (2026-07-09)
 
 CLI that reads a JSON archive of negative-result entries and checks each entry

schemas/calibration_audit.schema.json

@@ -0,0 +1,76 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "OpenAMP Calibration Audit Report",
+  "description": "Consistency audit across the calibration pipeline — intake report, gate verdict, engine proposal, and combined recalibration report.",
+  "type": "object",
+  "required": [
+    "report_type",
+    "schema_version",
+    "timestamp",
+    "artifacts_checked",
+    "overall_pass",
+    "checks",
+    "summary"
+  ],
+  "properties": {
+    "report_type": {
+      "type": "string",
+      "enum": ["calibration_audit"],
+      "description": "Fixed discriminator for this report type."
+    },
+    "schema_version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+$",
+      "description": "Version of this schema."
+    },
+    "timestamp": {
+      "type": "string",
+      "description": "ISO-8601 timestamp when the audit was generated."
+    },
+    "artifacts_checked": {
+      "type": "array",
+      "items": { "type": "string", "enum": ["intake", "gate", "engine", "report"] },
+      "description": "Which artifacts were available and checked."
+    },
+    "overall_pass": {
+      "type": "boolean",
+      "description": "True only if every check passed."
+    },
+    "checks": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["check_id", "description", "passed", "observed", "expected", "severity"],
+        "properties": {
+          "check_id": { "type": "string" },
+          "description": { "type": "string" },
+          "passed": { "type": "boolean" },
+          "observed": { "type": "string" },
+          "expected": { "type": "string" },
+          "severity": { "type": "string", "enum": ["error", "warning", "info"] }
+        }
+      },
+      "minItems": 1
+    },
+    "n_checks": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "n_passed": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "n_failed": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "n_warnings": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "summary": {
+      "type": "string",
+      "description": "Human-readable audit summary."
+    }
+  }
+}