diff --git a/Makefile b/Makefile index 1a3f1ff3..a40a1bde 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -900,3 +900,8 @@ pilot-package-check: .venv/bin/openamp-foundry pilot-package-check \ --entry-json '{"package_id":"PKG-001","batch_id":"BATCH-001","submission_date":"2026-07-09","pipeline_version":"0.8.1","included_artifacts":["selection_rationale","batch_priority","evidence_certificate"],"missing_artifacts":[],"reviewer":"alice","approver":"bob","completeness_score":1.0,"ready_to_submit":true,"dry_lab_only":true}' \ --format text + +calibration-intake-check: + .venv/bin/openamp-foundry calibration-intake-check \ + --entry-json '{"intake_id":"CAL-001","batch_id":"BATCH-001","candidate_id":"AMP-001","pipeline_version":"0.8.1","assay_type":"mic_assay","predicted_outcome":"active","observed_outcome":"active","predicted_confidence":0.85,"intake_date":"2026-07-09","reviewer":"alice","dry_lab_only":false}' \ + --format text diff --git a/docs/evidence/CALIBRATION_INTAKE_GUIDE.md b/docs/evidence/CALIBRATION_INTAKE_GUIDE.md new file mode 100644 index 00000000..c2f800de --- /dev/null +++ b/docs/evidence/CALIBRATION_INTAKE_GUIDE.md @@ -0,0 +1,65 @@ +# Post-Experiment Calibration Intake Guide + +When an external lab returns results from a pilot batch, each experimental outcome +must be compared against the pipeline's dry-lab predictions. This guide defines the +structured intake format for capturing that comparison and feeding it back into the +calibration system. + +## Purpose + +Calibration intake records exist so the pipeline can: +- detect when prediction confidence is systematically over- or under-estimated, +- trigger recalibration when error rates cross defined thresholds, and +- maintain an auditable trail of prediction-vs-outcome comparisons. + +## Required fields + +| Field | Type | Constraint | +|---|---|---| +| `intake_id` | str | Must start with `"CAL-"` | +| `batch_id` | str | Non-empty | +| `candidate_id` | str | Non-empty | +| `pipeline_version` | str | Non-empty | +| `assay_type` | str | One of `VALID_ASSAY_TYPES` | +| `predicted_outcome` | str | One of `VALID_OUTCOME_VALUES` | +| `observed_outcome` | str | One of `VALID_OUTCOME_VALUES` | +| `predicted_confidence` | float | 0.0 – 1.0 | +| `intake_date` | str | ISO 8601 `YYYY-MM-DD` | +| `reviewer` | str | Non-empty | +| `dry_lab_only` | bool | Must be `False` — this record captures real lab results | + +## Note on dry_lab_only + +Unlike all other records in the pipeline, a calibration intake record **must** set +`dry_lab_only = False` because it contains actual experimental outcome data. +Keeping `dry_lab_only = True` on a record that contains lab results would be a +false claim. The validator enforces this. + +## Valid assay types + +`mic_assay`, `hemolysis_assay`, `cytotoxicity_assay`, `membrane_disruption_assay`, `stability_assay` + +## Valid outcome values + +`active`, `inactive`, `inconclusive`, `not_tested` + +## Warnings + +- `predicted_outcome == observed_outcome` (correct prediction) — no warnings, just noted in PASS output. +- `predicted_confidence >= 0.90` and prediction is wrong — warning: high-confidence misprediction. +- `observed_outcome == "inconclusive"` — warning: result is inconclusive; do not count as calibration signal. + +## How to validate + +```bash +make calibration-intake-check +# or +openamp-foundry calibration-intake-check \ + --entry-json '{"intake_id":"CAL-001","batch_id":"BATCH-001","candidate_id":"AMP-001","pipeline_version":"0.8.1","assay_type":"mic_assay","predicted_outcome":"active","observed_outcome":"active","predicted_confidence":0.85,"intake_date":"2026-07-09","reviewer":"alice","dry_lab_only":false}' \ + --format text +``` + +## Honest-use boundary + +Calibration intake records describe real experimental outcomes. They must never be +confused with dry-lab predictions. `dry_lab_only` must be `False` on every intake record. diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 230a330e..712255a7 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase K K3 — pilot package completeness checker — v0.8.1) +> **Last updated:** 2026-07-09 (Phase K K4 — post-experiment calibration intake schema — v0.8.2) +> **New in v0.8.2:** Post-experiment calibration intake schema (K4 — Loop 122) — docs/evidence/CALIBRATION_INTAKE_GUIDE.md with purpose, required field table (11 fields), note on dry_lab_only=False (real lab results), valid assay types (5), valid outcome values (4), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/calibration_intake.py with CalibrationIntakeEntry dataclass (11 fields: intake_id, batch_id, candidate_id, pipeline_version, assay_type, predicted_outcome, observed_outcome, predicted_confidence, intake_date, reviewer, dry_lab_only=False), CalibrationIntakeResult dataclass (6 fields: intake_id, candidate_id, prediction_correct, passed, errors, warnings, dry_lab_only=False), VALID_ASSAY_TYPES (5: cytotoxicity_assay, hemolysis_assay, membrane_disruption_assay, mic_assay, stability_assay), VALID_OUTCOME_VALUES (4: active, inactive, inconclusive, not_tested), validate_calibration_intake() (11 checks: CAL- prefix, non-empty batch_id/candidate_id/pipeline_version/reviewer, valid assay_type, valid predicted_outcome, valid observed_outcome, confidence 0.0-1.0, YYYY-MM-DD date, dry_lab_only must be False; high-confidence misprediction warns, inconclusive observed warns), validate_calibration_intake_dict() (10 required fields guard). CLI (openamp-foundry calibration-intake-check) with --entry-json, --format text|json. make calibration-intake-check target. Phase K K4: prediction-vs-outcome comparisons are now machine-validated with honest dry_lab_only=False enforcement. > **New in v0.8.1:** Pilot package completeness checker (K3 — Loop 121) — docs/evidence/PILOT_PACKAGE_GUIDE.md with purpose, required field table (11 fields), mandatory artifact types table (3 types), valid artifact types, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/pilot_package.py with PilotPackageEntry dataclass (11 fields: package_id, batch_id, submission_date, pipeline_version, included_artifacts, missing_artifacts, reviewer, approver, completeness_score, ready_to_submit, dry_lab_only), PilotPackageResult dataclass (5 fields, dry_lab_only=True), MINIMUM_REQUIRED_ARTIFACTS (3), READINESS_SCORE_THRESHOLD (0.80), MANDATORY_ARTIFACT_TYPES (3: batch_priority, evidence_certificate, selection_rationale), VALID_ARTIFACT_TYPES (8 types), validate_pilot_package() (11 checks: PKG- prefix, non-empty batch_id/pipeline_version/reviewer/approver, YYYY-MM-DD submission_date, included_artifacts>=3, mandatory types covered, completeness_score 0.0-1.0, ready_to_submit must be False when score<0.80, dry_lab_only must be True; missing_artifacts non-empty warns, score<0.90 warns, same reviewer/approver warns), validate_pilot_package_dict() (10 required fields guard). CLI (openamp-foundry pilot-package-check) with --entry-json, --format text|json. make pilot-package-check target. Phase K K3: every pilot submission is now machine-validated for completeness. > > **New in v0.8.0:** Batch experiment priority ranker (K2 — Loop 120) — docs/evidence/BATCH_PRIORITY_GUIDE.md with purpose, required field table (11 fields), how-to-validate instructions. src/openamp_foundry/evidence/batch_priority.py with BatchPriorityEntry dataclass (12 fields: priority_id, batch_id, candidate_id, pipeline_version, priority_rank, priority_score, evidence_level, synthesis_complexity, novelty_tier, primary_rationale, disqualifying_concerns, dry_lab_only), BatchPriorityResult dataclass (6 fields, dry_lab_only=True), VALID_SYNTHESIS_COMPLEXITIES (3: high, low, medium), VALID_NOVELTY_TIERS (3: high, low, medium), VALID_EVIDENCE_LEVELS (6: 1-6), validate_batch_priority() (11 checks: PRI- prefix, non-empty batch_id/candidate_id/pipeline_version, priority_rank>=1, priority_score 0.0-1.0, valid evidence_level, valid synthesis_complexity, valid novelty_tier, non-empty primary_rationale, dry_lab_only must be True; evidence_level<=2 warns, rank==1+high-complexity warns, priority_score<0.30 warns), validate_batch_priority_dict() (10 required fields guard). CLI (openamp-foundry batch-priority-check) with --entry-json, --format text|json. make batch-priority-check target. Phase K K2: synthesis wave ranking is now machine-validated. diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 6f5ed7bd..8a8c4a4e 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,8 @@ # Roadmap +### v0.8.2 +- Phase K K4: post-experiment calibration intake schema — captures structured comparison of pipeline dry-lab prediction against actual experimental outcome. CalibrationIntakeEntry dataclass (dry_lab_only=False enforced), validate_calibration_intake(), CLI calibration-intake-check. + ## v0.8.1 — Loop 121: Phase K K3 — Pilot Package Completeness Checker `docs/evidence/PILOT_PACKAGE_GUIDE.md` with purpose, required field table (11 diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index a8f9c97f..800a349d 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -2919,3 +2919,38 @@ def _run_pilot_package_check(args: argparse.Namespace) -> int: print(" Pilot package validated. All required artifacts present.") return 0 if result.passed else 3 + + +def _run_calibration_intake_check(args: argparse.Namespace) -> int: + """Validate a post-experiment calibration intake entry.""" + import json as _json + from openamp_foundry.evidence.calibration_intake import validate_calibration_intake_dict + + try: + d = _json.loads(args.entry_json) + except _json.JSONDecodeError as exc: + print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"})) + return 2 + + if not isinstance(d, dict): + print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"})) + return 2 + + result = validate_calibration_intake_dict(d) + output_format = getattr(args, "format", "text") + + if output_format == "json": + import dataclasses + print(_json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + correct = "correct" if result.prediction_correct else "incorrect" + print(f"Calibration intake {result.intake_id} (candidate {result.candidate_id}, prediction {correct}): {status}") + for e in result.errors: + print(f" ERROR: {e}") + for w in result.warnings: + print(f" WARN: {w}") + if result.passed: + print(" Calibration intake validated. Prediction-vs-outcome comparison is recorded.") + + return 0 if result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 5cd7eb3f..49e477ef 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -15,7 +15,7 @@ from openamp_foundry.cli.commands.reports import _run_security_report_check from openamp_foundry.cli.commands.reports import _run_citation_check from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check -from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check +from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check import argparse import json @@ -2035,6 +2035,24 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Calibration intake check (Phase K K4) ──────────────────────── + cic = sub.add_parser( + "calibration-intake-check", + help=( + "Validate a post-experiment calibration intake entry. Compares pipeline " + "prediction against actual lab outcome and records the calibration signal." + ), + ) + cic.add_argument( + "--entry-json", type=str, required=True, + help="JSON string of a CalibrationIntakeEntry dict (required).", + ) + cic.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", @@ -2349,6 +2367,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "pilot-package-check": return _run_pilot_package_check(args) + if args.command == "calibration-intake-check": + return _run_calibration_intake_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/evidence/calibration_intake.py b/src/openamp_foundry/evidence/calibration_intake.py new file mode 100644 index 00000000..de57bce5 --- /dev/null +++ b/src/openamp_foundry/evidence/calibration_intake.py @@ -0,0 +1,180 @@ +"""Post-experiment calibration intake schema (Phase K K4). + +Captures the structured comparison of a pipeline dry-lab prediction +against an actual experimental outcome returned by an external lab. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import List + +VALID_ASSAY_TYPES: set[str] = { + "cytotoxicity_assay", + "hemolysis_assay", + "membrane_disruption_assay", + "mic_assay", + "stability_assay", +} + +VALID_OUTCOME_VALUES: set[str] = { + "active", + "inactive", + "inconclusive", + "not_tested", +} + +_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass +class CalibrationIntakeEntry: + """One post-experiment calibration intake record.""" + + intake_id: str + batch_id: str + candidate_id: str + pipeline_version: str + assay_type: str + predicted_outcome: str + observed_outcome: str + predicted_confidence: float + intake_date: str + reviewer: str + dry_lab_only: bool = False + + +@dataclass +class CalibrationIntakeResult: + """Validation result for a CalibrationIntakeEntry.""" + + intake_id: str + candidate_id: str + prediction_correct: bool + passed: bool + errors: List[str] + warnings: List[str] + dry_lab_only: bool = False + + +def validate_calibration_intake(entry: CalibrationIntakeEntry) -> CalibrationIntakeResult: + """Validate a CalibrationIntakeEntry. Returns a CalibrationIntakeResult.""" + errors: List[str] = [] + warnings: List[str] = [] + + if not entry.intake_id.startswith("CAL-"): + errors.append("intake_id must start with 'CAL-'") + + if not entry.batch_id: + errors.append("batch_id must not be empty") + + if not entry.candidate_id: + errors.append("candidate_id must not be empty") + + if not entry.pipeline_version: + errors.append("pipeline_version must not be empty") + + if entry.assay_type not in VALID_ASSAY_TYPES: + errors.append( + f"assay_type '{entry.assay_type}' is not valid; " + f"must be one of {sorted(VALID_ASSAY_TYPES)}" + ) + + if entry.predicted_outcome not in VALID_OUTCOME_VALUES: + errors.append( + f"predicted_outcome '{entry.predicted_outcome}' is not valid; " + f"must be one of {sorted(VALID_OUTCOME_VALUES)}" + ) + + if entry.observed_outcome not in VALID_OUTCOME_VALUES: + errors.append( + f"observed_outcome '{entry.observed_outcome}' is not valid; " + f"must be one of {sorted(VALID_OUTCOME_VALUES)}" + ) + + if not (0.0 <= entry.predicted_confidence <= 1.0): + errors.append("predicted_confidence must be between 0.0 and 1.0") + + if not _DATE_RE.match(entry.intake_date): + errors.append("intake_date must be YYYY-MM-DD") + + if not entry.reviewer: + errors.append("reviewer must not be empty") + + if entry.dry_lab_only is not False: + errors.append( + "dry_lab_only must be False for calibration intake records " + "(they contain real experimental outcomes)" + ) + + prediction_correct = entry.predicted_outcome == entry.observed_outcome + + if ( + not prediction_correct + and entry.predicted_confidence >= 0.90 + and not errors + ): + warnings.append( + f"High-confidence misprediction: predicted '{entry.predicted_outcome}' " + f"(confidence {entry.predicted_confidence:.2f}) but observed " + f"'{entry.observed_outcome}'. Review model calibration." + ) + + if entry.observed_outcome == "inconclusive" and not errors: + warnings.append( + "observed_outcome is 'inconclusive'; this result should not be " + "counted as a calibration signal until resolved." + ) + + return CalibrationIntakeResult( + intake_id=entry.intake_id, + candidate_id=entry.candidate_id, + prediction_correct=prediction_correct, + passed=len(errors) == 0, + errors=errors, + warnings=warnings, + dry_lab_only=False, + ) + + +def validate_calibration_intake_dict(d: dict) -> CalibrationIntakeResult: + """Validate a dict representation of a CalibrationIntakeEntry.""" + required = [ + "intake_id", + "batch_id", + "candidate_id", + "pipeline_version", + "assay_type", + "predicted_outcome", + "observed_outcome", + "predicted_confidence", + "intake_date", + "reviewer", + ] + for key in required: + if key not in d: + return CalibrationIntakeResult( + intake_id=d.get("intake_id", ""), + candidate_id=d.get("candidate_id", ""), + prediction_correct=False, + passed=False, + errors=[f"missing required field: {key}"], + warnings=[], + dry_lab_only=False, + ) + + entry = CalibrationIntakeEntry( + intake_id=d["intake_id"], + batch_id=d["batch_id"], + candidate_id=d["candidate_id"], + pipeline_version=d["pipeline_version"], + assay_type=d["assay_type"], + predicted_outcome=d["predicted_outcome"], + observed_outcome=d["observed_outcome"], + predicted_confidence=d["predicted_confidence"], + intake_date=d["intake_date"], + reviewer=d["reviewer"], + dry_lab_only=d.get("dry_lab_only", False), + ) + return validate_calibration_intake(entry) diff --git a/tests/evidence/test_evidence_calibration_intake.py b/tests/evidence/test_evidence_calibration_intake.py new file mode 100644 index 00000000..e23afb41 --- /dev/null +++ b/tests/evidence/test_evidence_calibration_intake.py @@ -0,0 +1,284 @@ +"""Tests for post-experiment calibration intake schema (Phase K K4).""" + +import pytest +from openamp_foundry.evidence.calibration_intake import ( + VALID_ASSAY_TYPES, + VALID_OUTCOME_VALUES, + CalibrationIntakeEntry, + CalibrationIntakeResult, + validate_calibration_intake, + validate_calibration_intake_dict, +) + + +def _valid_entry(**kwargs) -> CalibrationIntakeEntry: + defaults = dict( + intake_id="CAL-001", + batch_id="BATCH-001", + candidate_id="AMP-001", + pipeline_version="0.8.1", + assay_type="mic_assay", + predicted_outcome="active", + observed_outcome="active", + predicted_confidence=0.85, + intake_date="2026-07-09", + reviewer="alice", + dry_lab_only=False, + ) + defaults.update(kwargs) + return CalibrationIntakeEntry(**defaults) + + +# ── Constants ──────────────────────────────────────────────────────────────── + + +def test_valid_assay_types_contains_mic(): + assert "mic_assay" in VALID_ASSAY_TYPES + + +def test_valid_assay_types_contains_hemolysis(): + assert "hemolysis_assay" in VALID_ASSAY_TYPES + + +def test_valid_outcome_values_contains_active(): + assert "active" in VALID_OUTCOME_VALUES + + +def test_valid_outcome_values_contains_inconclusive(): + assert "inconclusive" in VALID_OUTCOME_VALUES + + +# ── Valid entry ─────────────────────────────────────────────────────────────── + + +def test_valid_entry_passes(): + result = validate_calibration_intake(_valid_entry()) + assert result.passed + assert result.errors == [] + + +def test_result_dry_lab_only_is_false(): + result = validate_calibration_intake(_valid_entry()) + assert result.dry_lab_only is False + + +def test_result_fields_match_entry(): + entry = _valid_entry() + result = validate_calibration_intake(entry) + assert result.intake_id == "CAL-001" + assert result.candidate_id == "AMP-001" + + +def test_prediction_correct_true_when_match(): + result = validate_calibration_intake( + _valid_entry(predicted_outcome="active", observed_outcome="active") + ) + assert result.prediction_correct is True + + +def test_prediction_correct_false_when_mismatch(): + result = validate_calibration_intake( + _valid_entry(predicted_outcome="active", observed_outcome="inactive") + ) + assert result.prediction_correct is False + + +def test_valid_with_all_assay_types(): + for assay in VALID_ASSAY_TYPES: + result = validate_calibration_intake(_valid_entry(assay_type=assay)) + assert result.passed, f"assay_type '{assay}' should be valid" + + +def test_valid_with_all_outcome_values(): + for outcome in VALID_OUTCOME_VALUES - {"inconclusive"}: + result = validate_calibration_intake( + _valid_entry(predicted_outcome=outcome, observed_outcome=outcome) + ) + assert result.passed, f"outcome '{outcome}' should be valid" + + +# ── intake_id validation ────────────────────────────────────────────────────── + + +def test_intake_id_missing_prefix_fails(): + result = validate_calibration_intake(_valid_entry(intake_id="001")) + assert not result.passed + assert any("CAL-" in e for e in result.errors) + + +def test_intake_id_wrong_prefix_fails(): + result = validate_calibration_intake(_valid_entry(intake_id="PKG-001")) + assert not result.passed + + +def test_intake_id_correct_prefix_passes(): + result = validate_calibration_intake(_valid_entry(intake_id="CAL-XYZ-99")) + assert result.passed + + +# ── assay_type validation ───────────────────────────────────────────────────── + + +def test_invalid_assay_type_fails(): + result = validate_calibration_intake(_valid_entry(assay_type="unknown_assay")) + assert not result.passed + assert any("assay_type" in e for e in result.errors) + + +# ── outcome validation ──────────────────────────────────────────────────────── + + +def test_invalid_predicted_outcome_fails(): + result = validate_calibration_intake(_valid_entry(predicted_outcome="maybe")) + assert not result.passed + assert any("predicted_outcome" in e for e in result.errors) + + +def test_invalid_observed_outcome_fails(): + result = validate_calibration_intake(_valid_entry(observed_outcome="unknown")) + assert not result.passed + assert any("observed_outcome" in e for e in result.errors) + + +# ── confidence validation ───────────────────────────────────────────────────── + + +def test_confidence_above_1_fails(): + result = validate_calibration_intake(_valid_entry(predicted_confidence=1.1)) + assert not result.passed + + +def test_confidence_negative_fails(): + result = validate_calibration_intake(_valid_entry(predicted_confidence=-0.1)) + assert not result.passed + + +def test_confidence_0_passes(): + result = validate_calibration_intake(_valid_entry(predicted_confidence=0.0)) + assert result.passed + + +def test_confidence_1_passes(): + result = validate_calibration_intake(_valid_entry(predicted_confidence=1.0)) + assert result.passed + + +# ── date validation ─────────────────────────────────────────────────────────── + + +def test_invalid_date_fails(): + result = validate_calibration_intake(_valid_entry(intake_date="07-09-2026")) + assert not result.passed + assert any("YYYY-MM-DD" in e for e in result.errors) + + +# ── dry_lab_only constraint ─────────────────────────────────────────────────── + + +def test_dry_lab_only_true_fails(): + result = validate_calibration_intake(_valid_entry(dry_lab_only=True)) + assert not result.passed + assert any("dry_lab_only" in e for e in result.errors) + + +# ── warnings ───────────────────────────────────────────────────────────────── + + +def test_high_confidence_misprediction_warns(): + result = validate_calibration_intake( + _valid_entry( + predicted_outcome="active", + observed_outcome="inactive", + predicted_confidence=0.95, + ) + ) + assert result.passed + assert any("High-confidence" in w or "misprediction" in w for w in result.warnings) + + +def test_inconclusive_observed_warns(): + result = validate_calibration_intake( + _valid_entry( + predicted_outcome="active", + observed_outcome="inconclusive", + ) + ) + assert result.passed + assert any("inconclusive" in w for w in result.warnings) + + +def test_correct_prediction_no_warnings(): + result = validate_calibration_intake( + _valid_entry(predicted_outcome="active", observed_outcome="active", predicted_confidence=0.95) + ) + assert result.passed + assert result.warnings == [] + + +def test_low_confidence_mismatch_no_warning(): + result = validate_calibration_intake( + _valid_entry( + predicted_outcome="active", + observed_outcome="inactive", + predicted_confidence=0.70, + ) + ) + assert result.passed + assert result.warnings == [] + + +# ── dict interface ──────────────────────────────────────────────────────────── + + +def test_dict_valid_passes(): + d = dict( + intake_id="CAL-D01", + batch_id="BATCH-D01", + candidate_id="AMP-D01", + pipeline_version="0.8.1", + assay_type="mic_assay", + predicted_outcome="active", + observed_outcome="active", + predicted_confidence=0.85, + intake_date="2026-07-09", + reviewer="alice", + dry_lab_only=False, + ) + result = validate_calibration_intake_dict(d) + assert result.passed + + +def test_dict_missing_field_fails(): + d = dict( + intake_id="CAL-D02", + batch_id="BATCH-D02", + candidate_id="AMP-D02", + pipeline_version="0.8.1", + assay_type="mic_assay", + predicted_outcome="active", + observed_outcome="active", + predicted_confidence=0.85, + # missing intake_date + reviewer="alice", + ) + result = validate_calibration_intake_dict(d) + assert not result.passed + assert any("intake_date" in e for e in result.errors) + + +def test_dict_dry_lab_only_defaults_false(): + d = dict( + intake_id="CAL-D03", + batch_id="BATCH-D03", + candidate_id="AMP-D03", + pipeline_version="0.8.1", + assay_type="mic_assay", + predicted_outcome="active", + observed_outcome="active", + predicted_confidence=0.85, + intake_date="2026-07-09", + reviewer="alice", + ) + result = validate_calibration_intake_dict(d) + assert result.passed + assert result.dry_lab_only is False diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 466d5625..70ec0e0e 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3780 +BASELINE = 3813 TOLERANCE = 0.05 # allow 5% fluctuation