From c932708251cf3628df5476b022752d06e66ff801 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 12:42:35 +0700 Subject: [PATCH] feat: Phase G G6 calibration-overfit warning -- prevents false learning on small cohorts --- Makefile | 8 +- docs/evidence/METRICS_CURRENT.md | 5 +- docs/research/ROADMAP.md | 52 ++++ schemas/calibration_overfit_check.schema.json | 53 ++++ src/openamp_foundry/calibration/__init__.py | 12 + .../calibration/overfit_warning.py | 214 +++++++++++++++ src/openamp_foundry/cli/commands/reports.py | 30 +++ src/openamp_foundry/cli/main.py | 49 +++- tests/calibration/test_overfit_warning.py | 252 ++++++++++++++++++ tests/test_test_count_regression.py | 4 +- 10 files changed, 673 insertions(+), 6 deletions(-) create mode 100644 schemas/calibration_overfit_check.schema.json create mode 100644 src/openamp_foundry/calibration/overfit_warning.py create mode 100644 tests/calibration/test_overfit_warning.py diff --git a/Makefile b/Makefile index b1f4f321..5d4cf25a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -672,3 +672,9 @@ negative-result-dashboard: --input examples/negative_result_dashboard_example.json \ --out-json outputs/negative_result_dashboard.json \ --out-md outputs/negative_result_dashboard.md + +calibration-overfit-check: + .venv/bin/openamp-foundry calibration-overfit-check \ + --cohort-sizes 8,25,50 --model-params 10 --n-features 5 \ + --out-json /tmp/overfit_check.json --out-md /tmp/overfit_check.md + @echo "Overfit check complete. See /tmp/overfit_check.json" diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 2b0424fb..4b93867d 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase G G5 — batch-2 selection rationale report — v0.5.83) +> **Last updated:** 2026-07-09 (Phase G G6 — calibration-overfit warning — v0.5.84) +> **New in v0.5.84:** Calibration-overfit warning (G6) — `check_cohort_overfit_risk()` and `run_overfit_check()` flag when a calibration cohort is too small relative to model parameters. Warns at three severity levels (critical/warning/caution/none). CLI (`openamp-foundry calibration-overfit-check`) accepts comma-separated cohort sizes, model params, n features. JSON + Markdown output. Schema (`schemas/calibration_overfit_check.schema.json`). `make calibration-overfit-check` target. 21 tests. 2995 total. > **New in v0.5.83:** Batch-2 selection rationale report (G5) — CLI (`openamp-foundry batch-rationale`) that generates a synthetic candidate pool, runs the batch-2 selector with configurable weights, and produces a per-candidate rationale report classifying each selected candidate into exploit / explore / diversity / combined roles. Reports weight configuration, role breakdown summary, per-candidate contributions (ensemble×weight, uncertainty×weight, diversity×weight), safety gate impact, and caveats. Schema (`schemas/batch_rationale_report.schema.json`). `make batch-rationale` target. 19 tests. 2974 total. > **New in v0.5.82:** Active-learning strategy comparison report (G4) — CLI (`openamp-foundry bench strategy-compare`) that compares 5 selection strategies (exploitation, exploration, diversity, combined, random) on the same synthetic pool with identical hidden active candidates. Each strategy runs multi-round recovery of hidden actives; the report ranks strategies by recall, compares the production selector vs pure strategies and random baseline, and produces structured JSON + Markdown output with caveats. Prevents one-selector bias by making strategy performance transparent. Schema (`schemas/active_learning_strategy_comparison.schema.json`). `make bench-strategy-compare` target. 18 tests. 2955 total. > **New in v0.5.81:** Calibration pipeline consistency audit (G3) — CLI (`openamp-foundry calibration-audit`) that checks consistency across calibration pipeline artifacts (intake report, gate verdict, engine proposal, recalibration report). Checks intake↔gate count matching, engine↔gate verdict agreement, engine L1 budget compliance, report↔gate verdict consistency, report↔engine proposal consistency, timestamp sanity, and cohort-metrics warnings. Schema (`schemas/calibration_audit.schema.json`). `make calibration-audit` and `make calibration-audit-example` targets. 18 tests. 2937 total. @@ -75,7 +76,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.83 (G5) +> **Pipeline version:** v0.5.84 (G6) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index d2ae59f0..7e91c2fe 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,57 @@ # Roadmap +## v0.5.84 — Loop 84: Phase G G6 — Calibration-Overfit Warning for Small Cohorts ✓ (2026-07-09) + +`check_cohort_overfit_risk()` and `run_overfit_check()` flag when a calibration +cohort is too small relative to model parameters. Prevents false learning from +under-powered cohorts. Warns at three severity levels (critical/warning/caution) +with human-readable messages and actionable recommendations. + +Changes: +- `src/openamp_foundry/calibration/overfit_warning.py` (G6) — + `check_cohort_overfit_risk()` assesses a single cohort's overfit risk from + cohort_size, model_params, n_features, and min_recommended threshold. Returns + warning_level (none/caution/warning/critical), ratio, message, and + recommendation. `run_overfit_check()` accepts a list of cohort sizes and + aggregates per-cohort results with worst_level, any_critical, any_warning flags. + `write_overfit_check_json()` and `write_overfit_check_markdown()` produce + structured output. +- `schemas/calibration_overfit_check.schema.json` (G6) — JSON Schema Draft + 2020-12 for the overfit check report. Validates per_cohort array, worst_level, + any_critical, any_warning, recommendation, dry_lab_only constraint. +- `src/openamp_foundry/calibration/__init__.py` — Exports + `check_cohort_overfit_risk`, `run_overfit_check`, `write_overfit_check_json`, + `write_overfit_check_markdown`. +- `src/openamp_foundry/cli/commands/reports.py` — Added + `_run_calibration_overfit_check()` CLI handler with `--cohort-sizes`, + `--model-params`, `--n-features`, `--min-recommended`, `--out-json`, + `--out-md` flags. +- `src/openamp_foundry/cli/main.py` — Registered `calibration-overfit-check` + subcommand with all argument flags and dispatch to handler. +- `Makefile` — Added `calibration-overfit-check` target with default params + writing to `/tmp/overfit_check.json` and `/tmp/overfit_check.md`. +- `tests/calibration/test_overfit_warning.py` — 21 tests covering: critical + threshold (<10), warning threshold (size < min_recommended AND ratio < 3.0), + caution (size < min_recommended OR ratio < 5.0), none level, run_overfit_check + mixed severity/all-none/all-critical, ratio calculation, message non-empty, + dry_lab_only=True everywhere, worst_level logic, any_critical/any_warning + flags, empty cohort list, single-cohort matching, ratio zero-division edge, + JSON writer, Markdown writer. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.84 G6 changelog. Test count: 2995. +- `tests/test_test_count_regression.py` — baseline updated to 2995. + +Honest boundaries: +- This check evaluates **statistical overfit risk** only. A passing check does + not confirm biological validity, safety, or real-world performance. +- Small cohorts can produce spurious correlations that appear significant in + dry-lab benchmarks. The warning is a computational safeguard, not a + biological guarantee. +- The severity thresholds (min_recommended=30, ratio<3.0 for warning, ratio<5.0 + for caution) are heuristic rules of thumb. Specific domains may require + stricter or looser thresholds. +- All calibration decisions require qualified human review regardless of + overfit check results. + ## v0.5.83 — Loop 83: Phase G G5 — Batch-2 Selection Rationale Report ✓ (2026-07-09) CLI (`openamp-foundry batch-rationale`) that generates a synthetic candidate pool, diff --git a/schemas/calibration_overfit_check.schema.json b/schemas/calibration_overfit_check.schema.json new file mode 100644 index 00000000..3df016dc --- /dev/null +++ b/schemas/calibration_overfit_check.schema.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OpenAMP Calibration Overfit Risk Check", + "description": "Assesses whether a calibration cohort is too small relative to model parameters — prevents false learning from under-powered cohorts. Dry-lab only.", + "type": "object", + "required": [ + "per_cohort", + "worst_level", + "any_critical", + "any_warning", + "recommendation", + "dry_lab_only" + ], + "properties": { + "per_cohort": { + "type": "array", + "items": { + "type": "object", + "required": [ + "cohort_size", + "model_params", + "n_features", + "ratio", + "warning_level", + "message", + "recommendation", + "dry_lab_only" + ], + "properties": { + "cohort_size": { "type": "integer", "minimum": 0 }, + "model_params": { "type": "integer", "minimum": 1 }, + "n_features": { "type": "integer", "minimum": 1 }, + "ratio": { "type": "number", "minimum": 0 }, + "warning_level": { + "type": "string", + "enum": ["none", "caution", "warning", "critical"] + }, + "message": { "type": "string", "minLength": 1 }, + "recommendation": { "type": "string", "minLength": 1 }, + "dry_lab_only": { "type": "boolean", "const": true } + } + } + }, + "worst_level": { + "type": "string", + "enum": ["none", "caution", "warning", "critical"] + }, + "any_critical": { "type": "boolean" }, + "any_warning": { "type": "boolean" }, + "recommendation": { "type": "string", "minLength": 1 }, + "dry_lab_only": { "type": "boolean", "const": true } + } +} diff --git a/src/openamp_foundry/calibration/__init__.py b/src/openamp_foundry/calibration/__init__.py index cca3705f..64a04493 100644 --- a/src/openamp_foundry/calibration/__init__.py +++ b/src/openamp_foundry/calibration/__init__.py @@ -61,6 +61,13 @@ write_gate_verdict_markdown, ) +from openamp_foundry.calibration.overfit_warning import ( + check_cohort_overfit_risk, + run_overfit_check, + write_overfit_check_json, + write_overfit_check_markdown, +) + from openamp_foundry.calibration.policy_version import ( VersionValidation, validate_policy_version, @@ -108,4 +115,9 @@ "validate_policy_version", "auto_increment_version", "VersionValidation", + # Overfit warning + "check_cohort_overfit_risk", + "run_overfit_check", + "write_overfit_check_json", + "write_overfit_check_markdown", ] diff --git a/src/openamp_foundry/calibration/overfit_warning.py b/src/openamp_foundry/calibration/overfit_warning.py new file mode 100644 index 00000000..4eed447a --- /dev/null +++ b/src/openamp_foundry/calibration/overfit_warning.py @@ -0,0 +1,214 @@ +"""Calibration overfit warning — flags when a cohort is too small +relative to model parameters to support reliable learning. + +This is a dry-lab computational check only. It does not measure +biological activity, safety, or real-world performance. +""" + +from __future__ import annotations + +import json +from pathlib import Path + + +def check_cohort_overfit_risk( + cohort_size: int, + model_params: int, + n_features: int, + min_recommended: int = 30, +) -> dict: + """Assess overfit risk for a single cohort. + + Parameters + ---------- + cohort_size : int + Number of samples in the calibration cohort. + model_params : int + Number of trainable parameters in the model. + n_features : int + Number of input features. + min_recommended : int + Minimum cohort size considered sufficient (default 30). + + Returns + ------- + dict with keys: cohort_size, model_params, n_features, ratio, + warning_level, message, recommendation, dry_lab_only. + """ + ratio = cohort_size / max(model_params, 1) + + if cohort_size < 10: + warning_level = "critical" + message = ( + f"Cohort size {cohort_size} is critically small — " + f"any apparent signal is likely noise." + ) + recommendation = ( + "Do not use this cohort for calibration. " + "Gather at least 10 samples before attempting any weight update." + ) + elif cohort_size < min_recommended and ratio < 3.0: + warning_level = "warning" + message = ( + f"Cohort size {cohort_size} is below the recommended minimum " + f"{min_recommended} and the ratio of samples to parameters " + f"({ratio:.2f}) is below 3.0. Risk of overfitting is high." + ) + recommendation = ( + "Consider pooling cohorts or deferring calibration " + "until more data are available." + ) + elif cohort_size < min_recommended or ratio < 5.0: + warning_level = "caution" + message = ( + f"Cohort size {cohort_size} or sample-to-parameter ratio " + f"({ratio:.2f}) suggests elevated overfit risk." + ) + recommendation = ( + "Proceed with caution. Monitor for score degradation " + "on held-out data after calibration." + ) + else: + warning_level = "none" + message = ( + f"Cohort size {cohort_size} and ratio {ratio:.2f} " + f"are adequate for calibration." + ) + recommendation = "No special action required." + + return { + "cohort_size": cohort_size, + "model_params": model_params, + "n_features": n_features, + "ratio": round(ratio, 4), + "warning_level": warning_level, + "message": message, + "recommendation": recommendation, + "dry_lab_only": True, + } + + +def run_overfit_check( + cohort_sizes: list[int], + model_params: int, + n_features: int, + min_recommended: int = 30, +) -> dict: + """Run overfit risk check across multiple cohorts. + + Parameters + ---------- + cohort_sizes : list of int + One cohort size per dataset fold, class, or replicate. + model_params : int + Number of trainable parameters in the model. + n_features : int + Number of input features. + min_recommended : int + Minimum cohort size considered sufficient (default 30). + + Returns + ------- + dict with keys: per_cohort, worst_level, any_critical, + any_warning, recommendation, dry_lab_only. + """ + if not cohort_sizes: + return { + "per_cohort": [], + "worst_level": "none", + "any_critical": False, + "any_warning": False, + "recommendation": "No cohorts provided. Nothing to check.", + "dry_lab_only": True, + } + + results = [ + check_cohort_overfit_risk(cs, model_params, n_features, min_recommended) + for cs in cohort_sizes + ] + + severity = {"none": 0, "caution": 1, "warning": 2, "critical": 3} + worst = max(results, key=lambda r: severity[r["warning_level"]]) + worst_level = worst["warning_level"] + any_critical = any(r["warning_level"] == "critical" for r in results) + any_warning = any(r["warning_level"] in ("warning", "critical") for r in results) + + if worst_level == "critical": + rec = ( + "Critical overfit risk detected in one or more cohorts. " + "Do not proceed with calibration until cohort sizes are increased." + ) + elif worst_level == "warning": + rec = ( + "Warning-level overfit risk detected. " + "Strongly consider pooling cohorts or deferring calibration." + ) + elif worst_level == "caution": + rec = ( + "Caution-level overfit risk detected. " + "Proceed with monitoring, but be aware of elevated false-learning risk." + ) + else: + rec = "No overfit risk detected across all cohorts." + + return { + "per_cohort": results, + "worst_level": worst_level, + "any_critical": any_critical, + "any_warning": any_warning, + "recommendation": rec, + "dry_lab_only": True, + } + + +def write_overfit_check_json(report: dict, path: str | Path) -> None: + """Write the overfit check report to a JSON file.""" + with open(path, "w") as f: + json.dump(report, f, indent=2) + + +def write_overfit_check_markdown(report: dict, path: str | Path) -> None: + """Write the overfit check report to a Markdown file.""" + lines = [ + "# Calibration Overfit Risk Report", + "", + "> **Dry-lab only.** This report is a computational check and does not", + "> measure biological activity, safety, or real-world performance.", + "", + "## Summary", + "", + f"| Key | Value |", + f"|-----|-------|", + f"| Worst level | {report['worst_level']} |", + f"| Any critical | {report['any_critical']} |", + f"| Any warning | {report['any_warning']} |", + ] + if report.get("recommendation"): + lines.append(f"| Recommendation | {report['recommendation']} |") + lines.append(f"| Dry-lab only | {report['dry_lab_only']} |") + lines.extend([ + "", + "## Per-Cohort Details", + "", + "| Cohort | Size | Params | Features | Ratio | Level | Message |", + "|--------|------|--------|----------|-------|-------|---------|", + ]) + for i, c in enumerate(report["per_cohort"]): + lines.append( + f"| {i} | {c['cohort_size']} | {c['model_params']} | " + f"{c['n_features']} | {c['ratio']} | {c['warning_level']} | " + f"{c['message']} |" + ) + + lines.extend([ + "", + "## Caveats", + "", + "- This check evaluates **statistical overfit risk** only.", + "- A passing check does not confirm biological validity.", + "- Small cohorts can produce spurious correlations that appear " + "significant in dry-lab benchmarks.", + "- All calibration decisions require qualified human review.", + ]) + with open(path, "w") as f: + f.write("\n".join(lines) + "\n") diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index c2d6bb02..10f3877b 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1000,3 +1000,33 @@ def _run_calibration_audit(args: argparse.Namespace) -> int: } print(json.dumps(cli_summary, indent=2)) return 0 if audit["overall_pass"] else 3 + + +def _run_calibration_overfit_check(args: argparse.Namespace) -> int: + from openamp_foundry.calibration.overfit_warning import ( + run_overfit_check, + write_overfit_check_json, + write_overfit_check_markdown, + ) + + report = run_overfit_check( + cohort_sizes=args.cohort_sizes, + model_params=args.model_params, + n_features=args.n_features, + min_recommended=args.min_recommended, + ) + if args.out_json: + write_overfit_check_json(report, args.out_json) + if args.out_md: + write_overfit_check_markdown(report, args.out_md) + + print(json.dumps({ + "status": "ok", + "worst_level": report["worst_level"], + "any_critical": report["any_critical"], + "any_warning": report["any_warning"], + "n_cohorts": len(report["per_cohort"]), + "out_json": args.out_json, + "out_md": args.out_md, + }, indent=2)) + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 5cceecd6..c5b21046 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1082,6 +1082,50 @@ def build_parser() -> argparse.ArgumentParser: help="ISO date (YYYY-MM-DD) for 'today'. Defaults to actual today.", ) + # ── Calibration overfit check ──────────────────────────────────── + calibration_overfit = sub.add_parser( + "calibration-overfit-check", + help=( + "Check whether calibration cohort sizes are adequate relative " + "to model parameters. Prevents false learning from under-powered " + "cohorts. Dry-lab only — does not measure biological activity." + ), + ) + calibration_overfit.add_argument( + "--cohort-sizes", + required=True, + type=lambda s: [int(x) for x in s.split(",")], + help="Comma-separated cohort sizes (e.g. '12,45,30').", + ) + calibration_overfit.add_argument( + "--model-params", + required=True, + type=int, + help="Number of trainable model parameters.", + ) + calibration_overfit.add_argument( + "--n-features", + required=True, + type=int, + help="Number of input features.", + ) + calibration_overfit.add_argument( + "--min-recommended", + type=int, + default=30, + help="Minimum recommended cohort size (default: 30).", + ) + calibration_overfit.add_argument( + "--out-json", + default=None, + help="Optional JSON output path.", + ) + calibration_overfit.add_argument( + "--out-md", + default=None, + help="Optional Markdown output path.", + ) + # ── Batch-2 rationale report ───────────────────────────────────── batch_rationale = sub.add_parser( "batch-rationale", @@ -1351,6 +1395,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "select-batch": return _run_select_batch(args) + if args.command == "calibration-overfit-check": + return _run_calibration_overfit_check(args) + parser.error("unknown command") return 2 diff --git a/tests/calibration/test_overfit_warning.py b/tests/calibration/test_overfit_warning.py new file mode 100644 index 00000000..87324455 --- /dev/null +++ b/tests/calibration/test_overfit_warning.py @@ -0,0 +1,252 @@ +"""Tests for calibration overfit warning module. + +Verifies: + - Critical threshold (cohort_size < 10) + - Warning threshold (cohort_size < min_recommended and ratio < 3.0) + - Caution threshold (cohort_size < min_recommended or ratio < 5.0) + - None level (cohort_size >= min_recommended and ratio >= 5.0) + - run_overfit_check with mixed severity + - run_overfit_check all-none + - run_overfit_check all-critical + - ratio calculation correctness + - message non-empty for each level + - dry_lab_only=True everywhere + - worst_level logic + - any_critical / any_warning flags + - Empty cohort list edge case + - JSON and Markdown writers produce output + - Single cohort matches check_cohort_overfit_risk + +Honest limitation: + These tests validate computational logic only. They do not measure + biological activity, safety, or real-world performance. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile + +import pytest + +from openamp_foundry.calibration.overfit_warning import ( + check_cohort_overfit_risk, + run_overfit_check, + write_overfit_check_json, + write_overfit_check_markdown, +) + + +class TestCheckCohortOverfitRisk: + """Single-cohort overfit risk assessment.""" + + def test_critical_cohort_size_less_than_10(self): + result = check_cohort_overfit_risk( + cohort_size=5, model_params=10, n_features=5 + ) + assert result["warning_level"] == "critical" + assert result["cohort_size"] == 5 + assert result["dry_lab_only"] is True + + def test_warning_threshold(self): + result = check_cohort_overfit_risk( + cohort_size=15, model_params=10, n_features=5, min_recommended=30 + ) + assert result["warning_level"] == "warning" + assert result["ratio"] < 3.0 + + def test_warning_cohort_below_min_recommended_with_low_ratio(self): + result = check_cohort_overfit_risk( + cohort_size=20, model_params=10, n_features=3, min_recommended=30 + ) + assert result["warning_level"] == "warning" + assert result["cohort_size"] < result.get("min_recommended", 30) + + def test_caution_cohort_small_but_ratio_ok(self): + result = check_cohort_overfit_risk( + cohort_size=20, model_params=4, n_features=3, min_recommended=30 + ) + assert result["warning_level"] == "caution" + assert result["ratio"] >= 3.0 + + def test_caution_ratio_low_but_cohort_ok(self): + result = check_cohort_overfit_risk( + cohort_size=35, model_params=10, n_features=5, min_recommended=30 + ) + assert result["warning_level"] == "caution" + assert result["cohort_size"] >= 30 + assert result["ratio"] < 5.0 + + def test_none_level(self): + result = check_cohort_overfit_risk( + cohort_size=100, model_params=10, n_features=5, min_recommended=30 + ) + assert result["warning_level"] == "none" + assert result["ratio"] >= 5.0 + + def test_all_messages_non_empty(self): + for cohort_size, mp, nf, mr in [ + (5, 10, 5, 30), + (15, 10, 5, 30), + (20, 4, 3, 30), + (100, 10, 5, 30), + ]: + result = check_cohort_overfit_risk( + cohort_size=cohort_size, model_params=mp, + n_features=nf, min_recommended=mr, + ) + assert len(result["message"]) > 0 + assert len(result["recommendation"]) > 0 + + def test_dry_lab_only_always_true(self): + for cohort_size in [5, 15, 25, 100]: + result = check_cohort_overfit_risk( + cohort_size=cohort_size, model_params=10, + n_features=5, min_recommended=30, + ) + assert result["dry_lab_only"] is True + + def test_ratio_calculation(self): + result = check_cohort_overfit_risk( + cohort_size=20, model_params=5, n_features=3 + ) + assert result["ratio"] == 4.0 + + def test_custom_min_recommended(self): + result = check_cohort_overfit_risk( + cohort_size=10, model_params=5, n_features=3, min_recommended=10 + ) + assert result["warning_level"] == "caution" + assert result["ratio"] == 2.0 + + +class TestRunOverfitCheck: + """Multi-cohort aggregated overfit check.""" + + def test_mixed_severity(self): + result = run_overfit_check( + cohort_sizes=[5, 25, 100], + model_params=10, + n_features=5, + min_recommended=30, + ) + assert len(result["per_cohort"]) == 3 + assert result["worst_level"] == "critical" + assert result["any_critical"] is True + assert result["any_warning"] is True + assert result["dry_lab_only"] is True + + def test_all_none(self): + result = run_overfit_check( + cohort_sizes=[100, 200, 300], + model_params=10, + n_features=5, + min_recommended=30, + ) + assert result["worst_level"] == "none" + assert result["any_critical"] is False + assert result["any_warning"] is False + assert all(c["warning_level"] == "none" for c in result["per_cohort"]) + + def test_all_critical(self): + result = run_overfit_check( + cohort_sizes=[3, 5, 8], + model_params=10, + n_features=5, + ) + assert result["worst_level"] == "critical" + assert result["any_critical"] is True + assert result["any_warning"] is True + assert all(c["warning_level"] == "critical" for c in result["per_cohort"]) + + def test_worst_level_logic(self): + result = run_overfit_check( + cohort_sizes=[25, 50], + model_params=10, + n_features=5, + min_recommended=30, + ) + levels = [c["warning_level"] for c in result["per_cohort"]] + assert "caution" in levels or "warning" in levels + + def test_any_critical_and_any_warning_flags(self): + result = run_overfit_check( + cohort_sizes=[8, 15, 100], + model_params=10, + n_features=5, + min_recommended=30, + ) + assert result["any_critical"] is True + assert result["any_warning"] is True + + def test_dry_lab_only_in_summary(self): + result = run_overfit_check( + cohort_sizes=[10, 20], model_params=10, n_features=5 + ) + assert result["dry_lab_only"] is True + for c in result["per_cohort"]: + assert c["dry_lab_only"] is True + + def test_empty_cohort_list(self): + result = run_overfit_check( + cohort_sizes=[], model_params=10, n_features=5 + ) + assert result["per_cohort"] == [] + assert result["worst_level"] == "none" + assert result["any_critical"] is False + assert result["any_warning"] is False + + def test_single_cohort_matches_check_function(self): + direct = check_cohort_overfit_risk(25, 10, 5, min_recommended=30) + aggregated = run_overfit_check( + [25], 10, 5, min_recommended=30 + ) + assert len(aggregated["per_cohort"]) == 1 + ac = aggregated["per_cohort"][0] + for key in ("cohort_size", "model_params", "n_features", + "ratio", "warning_level"): + assert ac[key] == direct[key] + + def test_ratio_zero_division_edge(self): + result = check_cohort_overfit_risk( + cohort_size=0, model_params=0, n_features=0 + ) + assert result["warning_level"] == "critical" + assert result["ratio"] == 0.0 + + +class TestWriters: + """JSON and Markdown output writers.""" + + def test_json_writer(self): + report = run_overfit_check([8, 25, 50], 10, 5) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as f: + path = f.name + try: + write_overfit_check_json(report, path) + with open(path) as f: + loaded = json.load(f) + assert loaded["worst_level"] == report["worst_level"] + assert loaded["any_critical"] == report["any_critical"] + assert len(loaded["per_cohort"]) == 3 + finally: + Path(path).unlink(missing_ok=True) + + def test_markdown_writer(self): + report = run_overfit_check([8, 25, 50], 10, 5) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".md", delete=False + ) as f: + path = f.name + try: + write_overfit_check_markdown(report, path) + with open(path) as f: + content = f.read() + assert "# Calibration Overfit Risk Report" in content + assert "Critical" in content or "critical" in content + assert "Dry-lab" in content + finally: + Path(path).unlink(missing_ok=True) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index e09b992f..85afa355 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 2974 tests as of Loop 83 (2026-07-09) -- Phase G G5 (batch-2 selection rationale report) +# Baseline: 2995 tests as of Loop 84 (2026-07-09) -- Phase G G6 (calibration-overfit warning) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 2974 +BASELINE = 2995 TOLERANCE = 0.05 # allow 5% fluctuation