diff --git a/Makefile b/Makefile index 366efcc3..2c6fb6ee 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -75,7 +75,9 @@ help: @echo " make validate-policy-version Validate proposed policy version against predecessor" @echo " make bump-policy-version Bump policy version with decision-log guard (requires --human-reviewer)" @echo " make generate-synthetic-lab-results Generate synthetic lab results for calibration testing" - @echo " make test Run full test suite (1803 passing tests, >=80% coverage)" + @echo " make calibration-audit-example Run calibration pipeline consistency audit on synthetic example" + @echo " make calibration-audit Run calibration pipeline consistency audit (INTAKE=[path] GATE=[path] ...)" + @echo " make test Run full test suite (2937 passing tests, >=80% coverage)" @echo " make coverage Test suite with per-module coverage report" @echo " make lint Ruff lint check on src/ tests/ scripts/" @echo " make typecheck mypy type check on src/" @@ -479,6 +481,32 @@ calibration-loop: --out-dir outputs/calibration_loop \ --seed 42 --n-batch-2 10 +calibration-audit-example: lab-result-intake-example + -PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli recalibration-gate \ + --intake-report outputs/calibration_intake_example.json \ + --intake-report-date 2026-07-04 \ + --out-json outputs/recalibration_gate_example.json \ + --out-md outputs/recalibration_gate_example.md 2>/dev/null + -PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli calibration-audit \ + --intake-report outputs/calibration_intake_example.json \ + --gate-verdict outputs/recalibration_gate_example.json \ + --out-json outputs/calibration_audit_example.json \ + --out-md outputs/calibration_audit_example.md + +calibration-audit: + @if [ -z "$(INTAKE)" ] && [ -z "$(GATE)" ] && [ -z "$(ENGINE)" ] && [ -z "$(REPORT)" ]; then \ + echo "Usage: make calibration-audit [INTAKE=] [GATE=] [ENGINE=] [REPORT=] [OUT_JSON=] [OUT_MD=]"; \ + echo "At least one artifact path is required."; \ + exit 1; \ + fi + PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli calibration-audit \ + $$(if [ -n "$(INTAKE)" ]; then echo --intake-report "$(INTAKE)"; fi) \ + $$(if [ -n "$(GATE)" ]; then echo --gate-verdict "$(GATE)"; fi) \ + $$(if [ -n "$(ENGINE)" ]; then echo --engine-proposal "$(ENGINE)"; fi) \ + $$(if [ -n "$(REPORT)" ]; then echo --recalibration-report "$(REPORT)"; fi) \ + $$(if [ -n "$(OUT_JSON)" ]; then echo --out-json "$(OUT_JSON)"; fi) \ + $$(if [ -n "$(OUT_MD)" ]; then echo --out-md "$(OUT_MD)"; fi) + generate-synthetic-lab-results: PYTHONPATH=src $(PYTHON) examples/lab_results_generator.py \ --panel-csv outputs/pilot_panel.csv \ diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index cb08b549..a49c6556 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase F F10 — negative-result archive completeness checker — v0.5.80) +> **Last updated:** 2026-07-09 (Phase G G3 — calibration pipeline consistency audit — v0.5.81) +> **New in v0.5.81:** Calibration pipeline consistency audit (G3) — CLI (`openamp-foundry calibration-audit`) that checks consistency across calibration pipeline artifacts (intake report, gate verdict, engine proposal, recalibration report). Checks intake↔gate count matching, engine↔gate verdict agreement, engine L1 budget compliance, report↔gate verdict consistency, report↔engine proposal consistency, timestamp sanity, and cohort-metrics warnings. Schema (`schemas/calibration_audit.schema.json`). `make calibration-audit` and `make calibration-audit-example` targets. 18 tests. 2937 total. > **New in v0.5.80:** Negative-result archive completeness checker (F10) — CLI (`scripts/check_negative_archive_completeness.py`) that reads a JSON archive of negative-result entries and checks each entry against completeness criteria: all required fields present, no duplicate candidate_ids, each entry has at least one content field (assay_result, score_safety, reviewer_notes, or reason_detail), date format valid YYYY-MM-DD, and optional intake_report_id references are well-formed. Outputs a structured completeness report as JSON and Markdown with summary, per-check results, and per-entry pass/fail. Schema (`schemas/negative_result_archive_completeness.schema.json`). Example file (`examples/negative_result_archive_example.json`) with 4 toy entries. 35 tests. 2919 total. > > **New in v0.5.79:** Negative-result dashboard (F9) — CLI (`scripts/negative_result_dashboard.py`) that reads a collection of negative-result entries from a JSON file and produces a structured dashboard with summary statistics (by category, by pipeline version), score distributions (activity, safety, novelty, ensemble), per-category cross-analysis, and pipeline insights (most common failure category, recalibration opportunities). Outputs JSON + Markdown. Example file (`examples/negative_result_dashboard_example.json`) with 15 toy entries across all 6 failure categories. Schema (`schemas/negative_result_dashboard.schema.json`). `make negative-result-dashboard` target. 33 tests. 2883 total. @@ -72,7 +73,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.79 (F9) +> **Pipeline version:** v0.5.81 (G3) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 6b1da9ca..38c09e78 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,55 @@ # Roadmap +## v0.5.81 — Loop 81: Phase G G3 — Calibration Pipeline Consistency Audit ✓ (2026-07-09) + +CLI (`openamp-foundry calibration-audit`) that checks consistency across the +calibration pipeline artifacts — intake report, gate verdict, engine proposal, +and combined recalibration report. Ensures a human reviewer inspecting the +calibration pipeline output can verify that all stages agree on candidate +counts, gate verdicts, weight proposals, and timestamps. + +Changes: +- `src/openamp_foundry/calibration/audit.py` (G3) — `run_calibration_audit()` + accepts file paths or pre-loaded dicts for any combination of the four + artifacts. Runs 12 consistency checks: artifact path existence, intake↔gate + count matching, engine↔gate verdict agreement, engine L1 budget compliance, + engine intake-link match, report↔gate verdict match, report↔engine proposal + match, timestamp sanity, and intake cohort-metrics warnings. Each check has + check_id, description, pass/fail, observed, expected, and severity (error/ + warning/info). Returns structured dict with overall_pass, checks array, and + summary. +- `schemas/calibration_audit.schema.json` (G3) — JSON Schema Draft 2020-12 for + the calibration audit report. Validates report_type, schema_version, timestamp, + artifacts_checked, overall_pass, checks array, and summary. +- `src/openamp_foundry/cli/commands/reports.py` — Added `_run_calibration_audit` + CLI handler with `--intake-report`, `--gate-verdict`, `--engine-proposal`, + `--recalibration-report`, `--out-json`, `--out-md` flags. +- `src/openamp_foundry/cli/main.py` — Registered `calibration-audit` subcommand + with all argument flags and dispatch to `_run_calibration_audit`. +- `Makefile` — Added `calibration-audit-example` and `calibration-audit` targets. + Example target runs on synthetic intake + gate outputs. +- `tests/calibration/test_calibration_audit.py` — 18 tests covering: no + artifact edge case, single artifact, intake↔gate count match/mismatch, + engine↔gate verdict match/mismatch, L1 budget within/exceeds, report↔gate + match, report↔engine match, future timestamp detection, cohort-metrics + warning, JSON schema conformance, Markdown output, synthetic example + consistency, synthetic path existence, engine without gate_passed, and + nonexistent path handling. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.81 G3 changelog. Test count: 2937. +- `tests/test_test_count_regression.py` — baseline updated to 2937. + +Honest boundaries: +- This audit checks **consistency** between pipeline artifacts, not biological + validity. A passing audit means the pipeline stages agreed with each other, + not that calibration decisions are correct. +- All tests use synthetic dict fixtures, not real wet-lab data. +- Timestamp checks detect future timestamps but do not validate chronological + ordering between artifacts (e.g., gate must precede engine). This is a known + limitation for a future iteration. +- The audit does not validate artifact schema conformance (each artifact's + schema validity is tested separately). It focuses on cross-artifact + consistency. + ## v0.5.80 — Loop 80: Phase F F10 — Negative-Result Archive Completeness Checker ✓ (2026-07-09) CLI that reads a JSON archive of negative-result entries and checks each entry diff --git a/schemas/calibration_audit.schema.json b/schemas/calibration_audit.schema.json new file mode 100644 index 00000000..77f2bdbb --- /dev/null +++ b/schemas/calibration_audit.schema.json @@ -0,0 +1,76 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OpenAMP Calibration Audit Report", + "description": "Consistency audit across the calibration pipeline — intake report, gate verdict, engine proposal, and combined recalibration report.", + "type": "object", + "required": [ + "report_type", + "schema_version", + "timestamp", + "artifacts_checked", + "overall_pass", + "checks", + "summary" + ], + "properties": { + "report_type": { + "type": "string", + "enum": ["calibration_audit"], + "description": "Fixed discriminator for this report type." + }, + "schema_version": { + "type": "string", + "pattern": "^\\d+\\.\\d+$", + "description": "Version of this schema." + }, + "timestamp": { + "type": "string", + "description": "ISO-8601 timestamp when the audit was generated." + }, + "artifacts_checked": { + "type": "array", + "items": { "type": "string", "enum": ["intake", "gate", "engine", "report"] }, + "description": "Which artifacts were available and checked." + }, + "overall_pass": { + "type": "boolean", + "description": "True only if every check passed." + }, + "checks": { + "type": "array", + "items": { + "type": "object", + "required": ["check_id", "description", "passed", "observed", "expected", "severity"], + "properties": { + "check_id": { "type": "string" }, + "description": { "type": "string" }, + "passed": { "type": "boolean" }, + "observed": { "type": "string" }, + "expected": { "type": "string" }, + "severity": { "type": "string", "enum": ["error", "warning", "info"] } + } + }, + "minItems": 1 + }, + "n_checks": { + "type": "integer", + "minimum": 0 + }, + "n_passed": { + "type": "integer", + "minimum": 0 + }, + "n_failed": { + "type": "integer", + "minimum": 0 + }, + "n_warnings": { + "type": "integer", + "minimum": 0 + }, + "summary": { + "type": "string", + "description": "Human-readable audit summary." + } + } +} diff --git a/src/openamp_foundry/calibration/audit.py b/src/openamp_foundry/calibration/audit.py new file mode 100644 index 00000000..40c303bf --- /dev/null +++ b/src/openamp_foundry/calibration/audit.py @@ -0,0 +1,625 @@ +"""Calibration pipeline consistency audit. + +Checks that the four artifacts produced across the calibration pipeline +(intake report, gate verdict, engine proposal, recalibration report) +are internally consistent and can be trusted by a human reviewer. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path +from typing import Any + + +_CHECK_DESCRIPTIONS: dict[str, str] = { + "INTAKE_GATE_COUNTS_MATCH": ( + "Candidate and lab-result counts must be identical between the " + "intake report and the gate verdict." + ), + "ENGINE_GATE_VERDICT_MATCH": ( + "The engine must respect the gate verdict: if may_recalibrate is " + "false, the engine must not produce a proposal." + ), + "ENGINE_L1_WITHIN_BUDGET": ( + "The engine's proposed L1 total must not exceed the policy budget." + ), + "ENGINE_INTAKE_LINK_MATCH": ( + "The intake report path referenced by the engine must match the " + "actual intake report used." + ), + "REPORT_GATE_VERDICT_MATCH": ( + "The combined recalibration report's gate-verdict section must " + "match the standalone gate verdict." + ), + "REPORT_ENGINE_PROPOSAL_MATCH": ( + "The combined recalibration report's proposal section must match " + "the standalone engine proposal." + ), + "INTAKE_PATH_PRESENT": ( + "The intake report path provided must point to an existing file." + ), + "GATE_PATH_PRESENT": ( + "The gate verdict path provided must point to an existing file." + ), + "ENGINE_PATH_PRESENT": ( + "The engine proposal path provided must point to an existing file." + ), + "REPORT_PATH_PRESENT": ( + "The combined recalibration report path provided must point to an " + "existing file." + ), + "TIMESTAMPS_REASONABLE": ( + "All artifact timestamps must be in the past (not in the future)." + ), + "INTAKE_COHORT_METRICS_WARN": ( + "If cohort metrics are flagged insufficient_data, the gate verdict " + "should reflect this limitation." + ), +} + +_CHECK_IDS = tuple(_CHECK_DESCRIPTIONS.keys()) + + +def _check( + check_id: str, + passed: bool, + observed: str, + expected: str, + severity: str = "error", +) -> dict[str, Any]: + return { + "check_id": check_id, + "description": _CHECK_DESCRIPTIONS.get(check_id, ""), + "passed": passed, + "observed": observed, + "expected": expected, + "severity": severity, + } + + +def _get_intake_counts( + intake: dict[str, Any], +) -> dict[str, int]: + return { + "n_panel_candidates": intake.get("n_panel_candidates", 0), + "n_matched_candidates": intake.get("n_matched_candidates", 0), + "n_lab_results": intake.get("n_lab_results", 0), + } + + +def _get_gate_counts(gate: dict[str, Any]) -> dict[str, int]: + return { + "n_panel_candidates": gate.get("n_panel_candidates", 0), + "n_matched_candidates": gate.get("n_matched_candidates", 0), + "n_lab_results": gate.get("n_lab_results", 0), + } + + +def _load_json(path: str | Path | None) -> dict[str, Any] | None: + if path is None: + return None + try: + with open(path) as f: + return json.load(f) + except (FileNotFoundError, json.JSONDecodeError, IsADirectoryError): + return None + + +def _artifact_by_name( + name: str, + mapping: dict[str, Any], +) -> Any: + return mapping.get(name) + + +def _fmt_counts(label: str, counts: dict[str, int]) -> str: + parts = [f"{k}={v}" for k, v in counts.items()] + return f"{label}: {{{', '.join(parts)}}}" + + +def _parse_timestamp(ts_str: str | None) -> datetime | None: + if not ts_str: + return None + try: + return datetime.fromisoformat(ts_str) + except (ValueError, TypeError): + return None + + +def run_calibration_audit( + intake_path: str | Path | None = None, + gate_path: str | Path | None = None, + engine_path: str | Path | None = None, + report_path: str | Path | None = None, + intake_data: dict[str, Any] | None = None, + gate_data: dict[str, Any] | None = None, + engine_data: dict[str, Any] | None = None, + report_data: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Run a consistency audit across the calibration pipeline artifacts. + + Accepts either file paths or pre-loaded dicts for each artifact. + When both are provided for the same artifact, the dict takes precedence. + """ + checks: list[dict[str, Any]] = [] + now = datetime.now() + + # Load artifacts: paths used for "present" checks, data for consistency + intake = intake_data if intake_data is not None else _load_json(intake_path) + gate = gate_data if gate_data is not None else _load_json(gate_path) + engine = engine_data if engine_data is not None else _load_json(engine_path) + report = report_data if report_data is not None else _load_json(report_path) + + # Determine available artifacts (from either data dicts or loaded files) + artifact_names = { + "intake": intake, + "gate": gate, + "engine": engine, + "report": report, + } + artifacts_checked = [name for name, data in artifact_names.items() if data is not None] + + # ── Artifact existence checks (only when paths are given) ────── + for name, path, check_id in [ + ("intake", intake_path, "INTAKE_PATH_PRESENT"), + ("gate", gate_path, "GATE_PATH_PRESENT"), + ("engine", engine_path, "ENGINE_PATH_PRESENT"), + ("report", report_path, "REPORT_PATH_PRESENT"), + ]: + if path is not None: + if Path(path).exists(): + checks.append( + _check( + check_id, + passed=True, + observed=f"{path} exists", + expected=f"{path} should exist", + ) + ) + else: + checks.append( + _check( + check_id, + passed=False, + observed=f"{path} does not exist", + expected=f"{path} should exist", + ) + ) + else: + if _artifact_by_name(name, artifact_names) is not None: + checks.append( + _check( + check_id, + passed=True, + observed=f"{name} provided as data dict (no path)", + expected=f"{name} should be available", + ) + ) + else: + checks.append( + _check( + check_id, + passed=False, + observed=f"no path or data provided for {name}", + expected=f"a file path or data dict for {name} must be provided", + severity="info", + ) + ) + + if not artifacts_checked: + overall_pass = False + summary = ( + "No artifacts were available for audit. Provide at least one " + "valid artifact path or data dict." + ) + return { + "report_type": "calibration_audit", + "schema_version": "1.0", + "timestamp": now.isoformat(timespec="seconds"), + "artifacts_checked": [], + "overall_pass": overall_pass, + "checks": checks, + "n_checks": len(checks), + "n_passed": sum(1 for c in checks if c["passed"]), + "n_failed": sum(1 for c in checks if not c["passed"]), + "n_warnings": sum(1 for c in checks if c["severity"] == "warning"), + "summary": summary, + } + + # ── Intake ↔ Gate count consistency ──────────────────────────── + if intake and gate and "intake" in artifacts_checked and "gate" in artifacts_checked: + intake_counts = _get_intake_counts(intake) + gate_counts = _get_gate_counts(gate) + counts_match = intake_counts == gate_counts + checks.append( + _check( + "INTAKE_GATE_COUNTS_MATCH", + passed=counts_match, + observed=( + f"{_fmt_counts('intake', intake_counts)}; " + f"{_fmt_counts('gate', gate_counts)}" + ), + expected="intake and gate counts must be identical", + ) + ) + elif "intake" in artifacts_checked or "gate" in artifacts_checked: + checks.append( + _check( + "INTAKE_GATE_COUNTS_MATCH", + passed=False, + observed="only one of intake/gate is available", + expected="both intake and gate are needed for count comparison", + severity="warning", + ) + ) + + # ── Engine ↔ Gate verdict consistency ────────────────────────── + if engine and gate and "engine" in artifacts_checked and "gate" in artifacts_checked: + gate_may = gate.get("may_recalibrate", False) + engine_gate = engine.get("gate_passed", None) + verdict_match = engine_gate is None or engine_gate == gate_may + checks.append( + _check( + "ENGINE_GATE_VERDICT_MATCH", + passed=verdict_match, + observed=( + f"gate may_recalibrate={gate_may}; " + f"engine gate_passed={engine_gate}" + ), + expected=( + "if engine records gate_passed, it must match " + "the gate's may_recalibrate" + ), + ) + ) + elif "engine" in artifacts_checked or "gate" in artifacts_checked: + checks.append( + _check( + "ENGINE_GATE_VERDICT_MATCH", + passed=False, + observed="only one of engine/gate is available", + expected="both engine and gate are needed for verdict comparison", + severity="warning", + ) + ) + + # ── Engine L1 budget consistency ─────────────────────────────── + if engine and "engine" in artifacts_checked: + l1_total = engine.get("l1_total", 0.0) + l1_budget = engine.get("l1_budget", 0.0) + l1_within = engine.get("l1_within_budget", False) + computed_within = l1_total <= l1_budget if l1_budget > 0 else True + l1_ok = l1_within == computed_within + checks.append( + _check( + "ENGINE_L1_WITHIN_BUDGET", + passed=l1_ok, + observed=( + f"l1_total={l1_total:.4f}, l1_budget={l1_budget:.4f}, " + f"declared l1_within_budget={l1_within}" + ), + expected=( + f"l1_total ({l1_total:.4f}) <= l1_budget " + f"({l1_budget:.4f}) and l1_within_budget=true" + ), + ) + ) + if not l1_ok: + checks[-1]["observed"] += ( + f" (computed: l1_total <= l1_budget is {computed_within})" + ) + + # ── Engine intake link consistency ───────────────────────────── + if engine and intake_path and "engine" in artifacts_checked: + engine_intake_ref = engine.get("intake_report", None) + resolved_ref = ( + str(Path(engine_intake_ref).resolve()) + if engine_intake_ref + else None + ) + resolved_provided = str(Path(intake_path).resolve()) if intake_path else None + link_match = ( + resolved_ref is None + or resolved_provided is None + or resolved_ref == resolved_provided + ) + checks.append( + _check( + "ENGINE_INTAKE_LINK_MATCH", + passed=link_match, + observed=( + f"engine intake_report={engine_intake_ref!r}; " + f"provided path={str(intake_path)!r}" + ), + expected="engine's intake_report should match the provided intake path", + severity="warning" if link_match else "error", + ) + ) + + # ── Report ↔ Gate verdict consistency ────────────────────────── + if report and gate and "report" in artifacts_checked and "gate" in artifacts_checked: + report_gv = report.get("gate_verdict", {}) + for key in ("may_recalibrate", "n_panel_candidates", "n_matched_candidates", "n_lab_results"): + r_val = report_gv.get(key) + g_val = gate.get(key) + gv_match = ( + report_gv.get("may_recalibrate") == gate.get("may_recalibrate") + and report_gv.get("n_panel_candidates") == gate.get("n_panel_candidates") + and report_gv.get("n_matched_candidates") == gate.get("n_matched_candidates") + and report_gv.get("n_lab_results") == gate.get("n_lab_results") + ) + checks.append( + _check( + "REPORT_GATE_VERDICT_MATCH", + passed=gv_match, + observed=( + f"report gate may_recalibrate={report_gv.get('may_recalibrate')}, " + f"n_panel={report_gv.get('n_panel_candidates')}, " + f"n_matched={report_gv.get('n_matched_candidates')}, " + f"n_lab={report_gv.get('n_lab_results')}; " + f"standalone gate may_recalibrate={gate.get('may_recalibrate')}" + ), + expected="report gate-verdict section must match standalone gate verdict", + ) + ) + elif "report" in artifacts_checked or "gate" in artifacts_checked: + checks.append( + _check( + "REPORT_GATE_VERDICT_MATCH", + passed=False, + observed="only one of report/gate is available", + expected="both report and gate are needed for comparison", + severity="warning", + ) + ) + + # ── Report ↔ Engine proposal consistency ─────────────────────── + if report and engine and "report" in artifacts_checked and "engine" in artifacts_checked: + r_prop = report.get("proposal", {}) + for key in ("gate_passed", "l1_total", "l1_budget", "l1_within_budget"): + r_val = r_prop.get(key) + e_val = engine.get(key) + prop_match = ( + r_prop.get("gate_passed") == engine.get("gate_passed") + and r_prop.get("l1_total") == engine.get("l1_total") + and r_prop.get("l1_budget") == engine.get("l1_budget") + and r_prop.get("l1_within_budget") == engine.get("l1_within_budget") + ) + checks.append( + _check( + "REPORT_ENGINE_PROPOSAL_MATCH", + passed=prop_match, + observed=( + f"report proposal gate_passed={r_prop.get('gate_passed')}, " + f"l1_total={r_prop.get('l1_total')}; " + f"engine gate_passed={engine.get('gate_passed')}, " + f"l1_total={engine.get('l1_total')}" + ), + expected=( + "report proposal section must match standalone engine proposal" + ), + ) + ) + elif "report" in artifacts_checked or "engine" in artifacts_checked: + checks.append( + _check( + "REPORT_ENGINE_PROPOSAL_MATCH", + passed=False, + observed="only one of report/engine is available", + expected="both report and engine are needed for comparison", + severity="warning", + ) + ) + + # ── Timestamp sanity ─────────────────────────────────────────── + ts_check_passed = True + ts_parts: list[str] = [] + for name, data in [ + ("intake", intake), + ("gate", gate), + ("engine", engine), + ("report", report), + ]: + if not data: + continue + ts_str = data.get("timestamp") or data.get("generated_at") + ts = _parse_timestamp(ts_str) + if ts and ts > now: + ts_check_passed = False + ts_parts.append(f"{name} timestamp {ts_str} is in the future") + elif ts: + ts_parts.append(f"{name} timestamp {ts_str} is OK") + else: + ts_parts.append(f"{name} has no parseable timestamp") + checks.append( + _check( + "TIMESTAMPS_REASONABLE", + passed=ts_check_passed, + observed="; ".join(ts_parts), + expected="all timestamps must be in the past", + ) + ) + + # ── Intake cohort metrics warning ────────────────────────────── + if intake and gate and "intake" in artifacts_checked and "gate" in artifacts_checked: + cohort_metrics = intake.get("cohort_metrics", {}) + any_insufficient = any( + m.get("insufficient_data", False) + for m in cohort_metrics.values() + ) + may_recalibrate = gate.get("may_recalibrate", False) + if any_insufficient and may_recalibrate: + checks.append( + _check( + "INTAKE_COHORT_METRICS_WARN", + passed=False, + observed=( + "intake cohort metrics have insufficient_data, " + "but gate verdict allows recalibration" + ), + expected=( + "gate should not allow recalibration when intake " + "cohort data is insufficient" + ), + severity="warning", + ) + ) + else: + checks.append( + _check( + "INTAKE_COHORT_METRICS_WARN", + passed=True, + observed=( + f"insufficient_data={any_insufficient}, " + f"may_recalibrate={may_recalibrate}" + ), + expected=( + "warn only when insufficient_data is true and " + "gate passes" + ), + severity="info", + ) + ) + + # ── Summarize ──────────────────────────────────────────────── + n_passed = sum(1 for c in checks if c["passed"]) + n_failed = sum(1 for c in checks if not c["passed"]) + n_warnings = sum(1 for c in checks if c["severity"] == "warning") + overall_pass = n_failed == 0 + + if not artifacts_checked: + summary = "No artifacts were checked." + elif overall_pass: + summary = ( + f"All {n_passed} check(s) passed across " + f"{len(artifacts_checked)} artifact(s): " + f"{', '.join(sorted(artifacts_checked))}." + ) + else: + summary = ( + f"{n_failed} of {len(checks)} check(s) failed across " + f"{len(artifacts_checked)} artifact(s): " + f"{', '.join(sorted(artifacts_checked))}. " + "Review the failed checks before trusting the pipeline output." + ) + + return { + "report_type": "calibration_audit", + "schema_version": "1.0", + "timestamp": now.isoformat(timespec="seconds"), + "artifacts_checked": sorted(set(artifacts_checked)), + "overall_pass": overall_pass, + "checks": checks, + "n_checks": len(checks), + "n_passed": n_passed, + "n_failed": n_failed, + "n_warnings": n_warnings, + "summary": summary, + } + + +def write_calibration_audit_json( + audit: dict[str, Any], + out_path: str | Path, +) -> Path: + """Write a calibration audit report to JSON.""" + p = Path(out_path) + p.parent.mkdir(parents=True, exist_ok=True) + with p.open("w") as f: + json.dump(audit, f, indent=2) + f.write("\n") + return p + + +def write_calibration_audit_markdown( + audit: dict[str, Any], + out_path: str | Path, +) -> Path: + """Write a calibration audit report as human-readable Markdown.""" + p = Path(out_path) + p.parent.mkdir(parents=True, exist_ok=True) + lines: list[str] = [] + + lines.append("# Calibration Pipeline Audit") + lines.append("") + + overall = "✅ PASS" if audit["overall_pass"] else "❌ FAIL" + lines.append(f"**Overall: {overall}**") + lines.append("") + lines.append(audit["summary"]) + lines.append("") + + lines.append("## Audit metadata") + lines.append("") + lines.append(f"- Report type: `{audit['report_type']}`") + lines.append(f"- Schema version: `{audit['schema_version']}`") + lines.append(f"- Generated at: **{audit['timestamp']}**") + lines.append(f"- Artifacts checked: {', '.join(audit['artifacts_checked'])}") + lines.append(f"- Checks: {audit['n_checks']} total, " + f"{audit['n_passed']} passed, " + f"{audit['n_failed']} failed, " + f"{audit['n_warnings']} warning(s)") + lines.append("") + + if not audit["checks"]: + lines.append("**No checks were performed.**") + lines.append("") + p.write_text("\n".join(lines)) + return p + + lines.append("## Consistency checks") + lines.append("") + lines.append("| ID | Description | Result | Observed | Expected | Severity |") + lines.append("|----|-------------|--------|----------|----------|----------|") + + for c in audit["checks"]: + icon = "✅" if c["passed"] else "❌" + lines.append( + f"| `{c['check_id']}` | {c['description']} | {icon} " + f"| `{c['observed']}` | {c['expected']} | {c['severity']} |" + ) + lines.append("") + + failed = [c for c in audit["checks"] if not c["passed"]] + if failed: + lines.append("## Failed checks") + lines.append("") + for c in failed: + lines.append(f"### {c['check_id']}") + lines.append("") + lines.append(f"- **Description:** {c['description']}") + lines.append(f"- **Observed:** `{c['observed']}`") + lines.append(f"- **Expected:** {c['expected']}") + lines.append(f"- **Severity:** {c['severity']}") + lines.append("") + + if audit["n_warnings"] > 0: + warnings = [c for c in audit["checks"] if c["severity"] == "warning"] + lines.append("## Warnings") + lines.append("") + for c in warnings: + lines.append(f"- **{c['check_id']}**: {c['description']}") + lines.append(f" - Observed: `{c['observed']}`") + lines.append(f" - Expected: {c['expected']}") + lines.append("") + + lines.append("## Limitations") + lines.append("") + lines.append( + "- This audit checks **consistency** between pipeline artifacts, " + "not the biological validity of the results." + ) + lines.append( + "- A passing audit means the pipeline stages agreed with each other, " + "not that the calibration decisions are biologically correct." + ) + lines.append( + "- Human review of the gate verdict, weight proposals, and " + "decision log remains mandatory." + ) + lines.append("") + + p.write_text("\n".join(lines)) + return p diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 85c06d17..c2d6bb02 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -964,3 +964,39 @@ def _run_validate_policy_version(args: argparse.Namespace) -> int: } print(json.dumps(payload, indent=2)) return 0 if result.passed else 3 + + +def _run_calibration_audit(args: argparse.Namespace) -> int: + """Run a consistency audit across the calibration pipeline artifacts.""" + from openamp_foundry.calibration.audit import ( + run_calibration_audit, + write_calibration_audit_json, + write_calibration_audit_markdown, + ) + + audit = run_calibration_audit( + intake_path=args.intake_report, + gate_path=args.gate_verdict, + engine_path=args.engine_proposal, + report_path=args.recalibration_report, + ) + + if args.out_json: + write_calibration_audit_json(audit, args.out_json) + if args.out_md: + write_calibration_audit_markdown(audit, args.out_md) + + cli_summary = { + "status": "ok" if audit["overall_pass"] else "issues_found", + "overall_pass": audit["overall_pass"], + "n_checks": audit["n_checks"], + "n_passed": audit["n_passed"], + "n_failed": audit["n_failed"], + "n_warnings": audit["n_warnings"], + "artifacts_checked": audit["artifacts_checked"], + "summary": audit["summary"], + "out_json": args.out_json, + "out_md": args.out_md, + } + print(json.dumps(cli_summary, indent=2)) + return 0 if audit["overall_pass"] else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 5dd0d5d0..5c481655 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -920,6 +920,46 @@ def build_parser() -> argparse.ArgumentParser: ), ) + calibration_audit = sub.add_parser( + "calibration-audit", + help=( + "Run a consistency audit across the calibration pipeline — " + "intake report, gate verdict, engine proposal, and combined " + "recalibration report. Checks that counts, verdicts, " + "proposals, and timestamps are internally consistent." + ), + ) + calibration_audit.add_argument( + "--intake-report", + default=None, + help="Path to the calibration intake JSON report.", + ) + calibration_audit.add_argument( + "--gate-verdict", + default=None, + help="Path to the recalibration gate verdict JSON.", + ) + calibration_audit.add_argument( + "--engine-proposal", + default=None, + help="Path to the recalibration engine proposal JSON.", + ) + calibration_audit.add_argument( + "--recalibration-report", + default=None, + help="Path to the combined recalibration report JSON.", + ) + calibration_audit.add_argument( + "--out-json", + default=None, + help="Optional output path for the audit report JSON.", + ) + calibration_audit.add_argument( + "--out-md", + default=None, + help="Optional output path for the audit report Markdown.", + ) + novelty_broad = sub.add_parser( "novelty-check-broad", help=( @@ -1184,6 +1224,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "validate-policy-version": return _run_validate_policy_version(args) + if args.command == "calibration-audit": + return _run_calibration_audit(args) + if args.command == "select-batch": return _run_select_batch(args) diff --git a/tests/calibration/test_calibration_audit.py b/tests/calibration/test_calibration_audit.py new file mode 100644 index 00000000..5e8cc350 --- /dev/null +++ b/tests/calibration/test_calibration_audit.py @@ -0,0 +1,421 @@ +"""Tests for the calibration pipeline audit. + +Verifies: + - Audit with no artifacts fails gracefully + - Audit with a single artifact produces partial coverage + - Intake ↔ gate count matching works (match and mismatch) + - Engine ↔ gate verdict matching works (match and mismatch) + - Engine L1 budget checks work + - Report ↔ gate verdict consistency works + - Report ↔ engine proposal consistency works + - Timestamp sanity checks detect future timestamps + - Intake cohort metrics warning triggers correctly + - JSON writer produces valid, schema-conformant output + - Markdown writer produces non-empty output + - Synthetic example data passes audit when all artifacts available + +Honest limitation: + These tests use synthetic dict fixtures, not real wet-lab data. +""" + +from __future__ import annotations + +import json +from datetime import datetime, timedelta +from pathlib import Path + +import pytest + +from openamp_foundry.calibration.audit import ( + run_calibration_audit, + write_calibration_audit_json, + write_calibration_audit_markdown, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCHEMA_PATH = REPO_ROOT / "schemas" / "calibration_audit.schema.json" +SYNTHETIC_INTAKE = REPO_ROOT / "outputs" / "calibration_intake_example.json" +SYNTHETIC_GATE = REPO_ROOT / "outputs" / "recalibration_gate_example.json" + + +# ── Fixtures ─────────────────────────────────────────────────────── + + +@pytest.fixture +def base_intake() -> dict: + return { + "n_panel_candidates": 8, + "n_matched_candidates": 4, + "n_lab_results": 5, + "n_orphan_lab_results": 0, + "cohort_metrics": {}, + "report_disclaimer": "test data", + } + + +@pytest.fixture +def base_gate() -> dict: + return { + "may_recalibrate": False, + "n_panel_candidates": 8, + "n_matched_candidates": 4, + "n_lab_results": 5, + "policy_version": 1, + "summary": "FAIL: test scenario", + } + + +@pytest.fixture +def base_engine() -> dict: + return { + "gate_passed": False, + "l1_total": 0.0, + "l1_budget": 0.10, + "l1_within_budget": True, + "deltas": [], + "notes": ["test only"], + } + + +@pytest.fixture +def base_report() -> dict: + now = datetime.now() + return { + "report_type": "recalibration_report", + "schema_version": "1.0", + "timestamp": now.isoformat(), + "policy_version": 1, + "human_review_required": True, + "gate_verdict": { + "may_recalibrate": False, + "n_panel_candidates": 8, + "n_matched_candidates": 4, + "n_lab_results": 5, + }, + "proposal": { + "gate_passed": False, + "l1_total": 0.0, + "l1_budget": 0.10, + "l1_within_budget": True, + "deltas": [], + "notes": ["test only"], + }, + } + + +# ── No-artifact edge cases ──────────────────────────────────────── + + +def test_audit_no_artifacts(): + """Audit with no paths and no data fails gracefully.""" + result = run_calibration_audit() + assert result["report_type"] == "calibration_audit" + assert result["overall_pass"] is False + assert result["artifacts_checked"] == [] + assert result["n_checks"] == 4 # four artifact-path checks + assert result["summary"] == "No artifacts were available for audit. " \ + "Provide at least one valid artifact path or data dict." + + +def test_audit_single_artifact_via_data(): + """Audit with a single data dict produces partial coverage.""" + result = run_calibration_audit(intake_data={"n_panel_candidates": 5}) + assert result["overall_pass"] is False + assert "intake" in result["artifacts_checked"] + # Should include existence check + partial comparison warnings + n_failed = result["n_failed"] + assert n_failed >= 1 + + +# ── Count consistency ────────────────────────────────────────────── + + +def test_intake_gate_counts_match(): + """When intake and gate counts match, the check passes.""" + intake = { + "n_panel_candidates": 10, + "n_matched_candidates": 6, + "n_lab_results": 8, + "n_orphan_lab_results": 0, + "cohort_metrics": {}, + } + gate = { + "may_recalibrate": False, + "n_panel_candidates": 10, + "n_matched_candidates": 6, + "n_lab_results": 8, + } + result = run_calibration_audit( + intake_data=intake, gate_data=gate, + ) + count_check = [c for c in result["checks"] if c["check_id"] == "INTAKE_GATE_COUNTS_MATCH"] + assert len(count_check) == 1 + assert count_check[0]["passed"] is True + + +def test_intake_gate_counts_mismatch(): + """When intake and gate counts differ, the check fails.""" + intake = { + "n_panel_candidates": 10, + "n_matched_candidates": 6, + "n_lab_results": 8, + "n_orphan_lab_results": 0, + "cohort_metrics": {}, + } + gate = { + "may_recalibrate": False, + "n_panel_candidates": 10, + "n_matched_candidates": 5, + "n_lab_results": 7, + } + result = run_calibration_audit( + intake_data=intake, gate_data=gate, + ) + count_check = [c for c in result["checks"] if c["check_id"] == "INTAKE_GATE_COUNTS_MATCH"] + assert len(count_check) == 1 + assert count_check[0]["passed"] is False + + +# ── Engine ↔ Gate verdict consistency ────────────────────────────── + + +def test_engine_gate_verdict_match(): + """Engine gate_passed should match gate may_recalibrate.""" + result = run_calibration_audit( + gate_data={"may_recalibrate": False}, + engine_data={"gate_passed": False, "l1_total": 0.0, "l1_budget": 0.10, "l1_within_budget": True}, + ) + match_check = [c for c in result["checks"] if c["check_id"] == "ENGINE_GATE_VERDICT_MATCH"] + assert len(match_check) == 1 + assert match_check[0]["passed"] is True + + +def test_engine_gate_verdict_mismatch(): + """Engine gate_passed disagrees with gate may_recalibrate → fail.""" + result = run_calibration_audit( + gate_data={"may_recalibrate": True}, + engine_data={"gate_passed": False, "l1_total": 0.0, "l1_budget": 0.10, "l1_within_budget": True}, + ) + match_check = [c for c in result["checks"] if c["check_id"] == "ENGINE_GATE_VERDICT_MATCH"] + assert len(match_check) == 1 + assert match_check[0]["passed"] is False + + +# ── L1 budget checks ────────────────────────────────────────────── + + +def test_engine_l1_within_budget(): + """L1 total within budget → check passes.""" + result = run_calibration_audit( + engine_data={"gate_passed": True, "l1_total": 0.05, "l1_budget": 0.10, "l1_within_budget": True}, + ) + budget_check = [c for c in result["checks"] if c["check_id"] == "ENGINE_L1_WITHIN_BUDGET"] + assert len(budget_check) == 1 + assert budget_check[0]["passed"] is True + + +def test_engine_l1_exceeds_budget(): + """L1 total exceeds budget → check fails.""" + result = run_calibration_audit( + engine_data={"gate_passed": True, "l1_total": 0.15, "l1_budget": 0.10, "l1_within_budget": True}, + ) + budget_check = [c for c in result["checks"] if c["check_id"] == "ENGINE_L1_WITHIN_BUDGET"] + assert len(budget_check) == 1 + assert budget_check[0]["passed"] is False + + +# ── Report consistency ───────────────────────────────────────────── + + +def test_report_gate_verdict_match(): + """Report gate-verdict section matches standalone gate.""" + report = { + "report_type": "recalibration_report", + "gate_verdict": { + "may_recalibrate": False, + "n_panel_candidates": 8, + "n_matched_candidates": 4, + "n_lab_results": 5, + }, + } + gate = { + "may_recalibrate": False, + "n_panel_candidates": 8, + "n_matched_candidates": 4, + "n_lab_results": 5, + } + result = run_calibration_audit(report_data=report, gate_data=gate) + rv_check = [c for c in result["checks"] if c["check_id"] == "REPORT_GATE_VERDICT_MATCH"] + assert len(rv_check) == 1 + assert rv_check[0]["passed"] is True + + +def test_report_engine_proposal_match(): + """Report proposal section matches standalone engine.""" + report = { + "report_type": "recalibration_report", + "gate_verdict": {"may_recalibrate": False, "n_panel_candidates": 0, "n_matched_candidates": 0, "n_lab_results": 0}, + "proposal": { + "gate_passed": False, + "l1_total": 0.0, + "l1_budget": 0.10, + "l1_within_budget": True, + }, + } + engine = { + "gate_passed": False, + "l1_total": 0.0, + "l1_budget": 0.10, + "l1_within_budget": True, + } + result = run_calibration_audit(report_data=report, engine_data=engine) + rp_check = [c for c in result["checks"] if c["check_id"] == "REPORT_ENGINE_PROPOSAL_MATCH"] + assert len(rp_check) == 1 + assert rp_check[0]["passed"] is True + + +# ── Timestamp sanity ────────────────────────────────────────────── + + +def test_timestamp_future_detected(): + """A future timestamp should fail the timestamp check.""" + future = (datetime.now() + timedelta(days=1)).isoformat() + intake = { + "timestamp": future, + "n_panel_candidates": 1, + "n_matched_candidates": 0, + "n_lab_results": 0, + "n_orphan_lab_results": 0, + "cohort_metrics": {}, + } + result = run_calibration_audit(intake_data=intake) + ts_check = [c for c in result["checks"] if c["check_id"] == "TIMESTAMPS_REASONABLE"] + assert len(ts_check) == 1 + assert ts_check[0]["passed"] is False + assert "future" in ts_check[0]["observed"].lower() + + +# ── Cohort metrics warning ───────────────────────────────────────── + + +def test_cohort_metrics_warn_on_insufficient_data_with_gate_pass(): + """When intake has insufficient_data but gate passes, warn.""" + intake = { + "n_panel_candidates": 5, + "n_matched_candidates": 2, + "n_lab_results": 2, + "n_orphan_lab_results": 0, + "cohort_metrics": { + "mic": { + "n": 2, + "min_required": 5, + "insufficient_data": True, + }, + }, + } + gate = { + "may_recalibrate": True, + "n_panel_candidates": 5, + "n_matched_candidates": 2, + "n_lab_results": 2, + } + result = run_calibration_audit(intake_data=intake, gate_data=gate) + warn_check = [c for c in result["checks"] if c["check_id"] == "INTAKE_COHORT_METRICS_WARN"] + assert len(warn_check) == 1 + assert warn_check[0]["passed"] is False + assert warn_check[0]["severity"] == "warning" + + +# ── JSON writer ──────────────────────────────────────────────────── + + +def test_json_writer_schema_conformant(tmp_path): + """Written JSON should be valid and schema-conformant.""" + out = tmp_path / "audit.json" + result = run_calibration_audit( + intake_data={"n_panel_candidates": 5, "n_matched_candidates": 2, "n_lab_results": 3, "n_orphan_lab_results": 0, "cohort_metrics": {}}, + ) + write_calibration_audit_json(result, out) + assert out.exists() + with open(out) as f: + written = json.load(f) + assert written["report_type"] == "calibration_audit" + assert "overall_pass" in written + assert "checks" in written + assert "summary" in written + + # Validate against schema + import jsonschema + with open(SCHEMA_PATH) as f: + schema = json.load(f) + jsonschema.Draft202012Validator(schema).validate(written) + + +# ── Markdown writer ──────────────────────────────────────────────── + + +def test_markdown_writer_non_empty(tmp_path): + """Written Markdown should be non-empty and readable.""" + out = tmp_path / "audit.md" + result = run_calibration_audit( + intake_data={"n_panel_candidates": 5, "n_matched_candidates": 2, "n_lab_results": 3, "n_orphan_lab_results": 0, "cohort_metrics": {}}, + ) + write_calibration_audit_markdown(result, out) + assert out.exists() + text = out.read_text() + assert len(text) > 100 + assert "Calibration Pipeline Audit" in text + assert "Consistency checks" in text + + +# ── Full synthetic example ───────────────────────────────────────── + + +def test_synthetic_example_intake_and_gate_consistent(): + """The synthetic intake and gate examples should be internally consistent.""" + with open(SYNTHETIC_INTAKE) as f: + intake = json.load(f) + with open(SYNTHETIC_GATE) as f: + gate = json.load(f) + result = run_calibration_audit(intake_data=intake, gate_data=gate) + # Count-specific checks should pass + count_check = [c for c in result["checks"] if c["check_id"] == "INTAKE_GATE_COUNTS_MATCH"] + assert count_check[0]["passed"] is True + # Cohort metrics should be flagged but gate should be false → no warning + warn_check = [c for c in result["checks"] if c["check_id"] == "INTAKE_COHORT_METRICS_WARN"] + assert warn_check[0]["passed"] is True + + +# ── Known artifact paths ─────────────────────────────────────────── + + +def test_synthetic_artifact_paths_exist(): + """The synthetic example output files should exist on disk.""" + assert SYNTHETIC_INTAKE.exists(), f"expected intake at {SYNTHETIC_INTAKE}" + assert SYNTHETIC_GATE.exists(), f"expected gate at {SYNTHETIC_GATE}" + + +# ── Engine without gate_passed field ─────────────────────────────── + + +def test_engine_without_gate_passed(): + """Engine with no gate_passed field should not trigger a verdict mismatch.""" + engine = {"l1_total": 0.05, "l1_budget": 0.10, "l1_within_budget": True, "deltas": []} + gate = {"may_recalibrate": True} + result = run_calibration_audit(gate_data=gate, engine_data=engine) + match_check = [c for c in result["checks"] if c["check_id"] == "ENGINE_GATE_VERDICT_MATCH"] + assert match_check[0]["passed"] is True # None == None is vacuously true + + +# ── File path existence checks via paths ─────────────────────────── + + +def test_audit_with_nonexistent_paths(tmp_path): + """Non-existent file paths should produce failed existence checks.""" + result = run_calibration_audit( + intake_path=str(tmp_path / "nonexistent_intake.json"), + gate_path=str(tmp_path / "nonexistent_gate.json"), + ) + exist_checks = {c["check_id"]: c for c in result["checks"]} + assert exist_checks["INTAKE_PATH_PRESENT"]["passed"] is False + assert exist_checks["GATE_PATH_PRESENT"]["passed"] is False diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 816e22ff..f8dfc2e1 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -4,7 +4,7 @@ # Baseline: 2919 tests as of Loop 80 (2026-07-09) -- Phase F F10 (negative-result archive completeness checker) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 2919 +BASELINE = 2937 TOLERANCE = 0.05 # allow 5% fluctuation