diff --git a/Makefile b/Makefile index 5fc2aa3e..883c4ded 100644 --- a/Makefile +++ b/Makefile @@ -974,3 +974,7 @@ dataset-release-check: .venv/bin/openamp-foundry dataset-release-check \ --entry-json '{"release_id":"DSR-001","dataset_name":"OpenAMP Wave 0.5 Candidates","dataset_version":"1.0.0","release_date":"2026-07-09","license_identifier":"CC-BY-4.0","data_sources":["UniProt AMP database","APD3","in-house pipeline"],"contains_sequences":true,"contains_activity_data":false,"dual_use_assessed":true,"usage_policy_url":"https://openamp.example.org/data-policy","contact_email":"data@openamp.example.org","release_approved":true,"dry_lab_only":true}' \ --format text + +.PHONY: calibration-improvement-check +calibration-improvement-check: + openamp-foundry calibration-improvement-check --entry-json '{"improvement_id":"CIR-TEST","pipeline_version_before":"v1.0","pipeline_version_after":"v1.1","trigger_ids":["CPS-001"],"action_taken":"threshold adjustment","action_category":"threshold_adjustment","brier_score_before":0.3,"brier_score_after":0.2,"improvement_confirmed":true,"reviewer":"test"}' diff --git a/docs/evidence/CALIBRATION_IMPROVEMENT_GUIDE.md b/docs/evidence/CALIBRATION_IMPROVEMENT_GUIDE.md new file mode 100644 index 00000000..69a688cc --- /dev/null +++ b/docs/evidence/CALIBRATION_IMPROVEMENT_GUIDE.md @@ -0,0 +1,49 @@ +# Calibration Improvement Record Guide + +Schema: `CalibrationImprovementEntry` — documents recalibration actions taken after drift or quality failures. + +## Purpose + +When calibration performance drops below threshold (high FP rate, low recall, poor Brier score) or +prediction drift is detected, a recalibration action must be taken. This schema provides an auditable +record of what changed, why, and what effect it had — closing the loop between detection and correction. + +## Fields + +| Field | Type | Description | +|-------|------|-------------| +| `improvement_id` | str | Unique ID, must start with "CIR-" | +| `pipeline_version_before` | str | Pipeline version before recalibration | +| `pipeline_version_after` | str | Pipeline version after recalibration | +| `trigger_ids` | List[str] | IDs of CPS- or DRM- records that triggered this action (at least 1) | +| `action_taken` | str | Description of what was changed (max 500 chars) | +| `action_category` | str | One of: threshold_adjustment, retraining, feature_removal, feature_addition, data_augmentation, weighting_change | +| `brier_score_before` | float | Brier score before action (0.0–1.0) | +| `brier_score_after` | float | Brier score after action (0.0–1.0) | +| `improvement_confirmed` | bool | True if brier_score_after < brier_score_before | +| `reviewer` | str | Who reviewed this improvement record | +| `dry_lab_only` | bool | False (may incorporate real lab feedback) | + +## Validation Rules + +1. `improvement_id` must start with "CIR-" +2. `pipeline_version_before` and `pipeline_version_after` must differ +3. `trigger_ids` must contain at least 1 entry +4. Each trigger_id must start with "CPS-" or "DRM-" +5. `action_taken` must not exceed 500 characters +6. `action_category` must be one of the 6 valid categories +7. `brier_score_before` and `brier_score_after` must be in [0.0, 1.0] +8. `improvement_confirmed` must be consistent: True iff brier_score_after < brier_score_before + +## Warning Conditions + +- brier_score_after >= brier_score_before but improvement_confirmed=True (inconsistency) +- brier_score_after > POOR_BRIER_THRESHOLD (0.25) even after recalibration +- pipeline_version_before == pipeline_version_after warns about missing version bump + +## Constants + +- `ACTION_TAKEN_MAX_LENGTH = 500` +- `VALID_ACTION_CATEGORIES = {"threshold_adjustment", "retraining", "feature_removal", "feature_addition", "data_augmentation", "weighting_change"}` +- `VALID_TRIGGER_PREFIXES = {"CPS-", "DRM-"}` +- `POOR_BRIER_THRESHOLD = 0.25` diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 1a649f05..d13fbd28 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,10 @@ # Roadmap +### v0.10.1 - Phase O O3: Calibration Improvement Record +- CalibrationImprovementEntry schema documents recalibration actions taken after drift/quality failures +- 6 action categories, trigger_ids linked to CPS-/DRM- records +- Closes the loop between detection and correction + ### v0.10.0 — Phase O O2: Prediction Drift Monitor - Added `PredictionDriftEntry` schema tracking mean/std score shifts between reference and evaluation batches - Detects silent model degradation before wet-lab confirmation diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index d36d9b70..223b9fc1 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -3457,3 +3457,34 @@ def _run_prediction_drift_check(args) -> int: print(f" WARN: {warn}") return 0 if result.passed else 1 + + +def _run_calibration_improvement_check(args) -> int: + import json + from openamp_foundry.evidence.calibration_improvement_record import ( + validate_calibration_improvement_dict, + ) + + if args.entry_json: + try: + data = json.loads(args.entry_json) + except json.JSONDecodeError as exc: + print(f"Error: invalid JSON: {exc}", file=__import__("sys").stderr) + return 1 + else: + data = json.load(__import__("sys").stdin) + + result = validate_calibration_improvement_dict(data) + + if args.format == "json": + import dataclasses + print(json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + print(f"[{status}] Calibration Improvement: {result.improvement_id}") + for err in result.errors: + print(f" ERROR: {err}") + for warn in result.warnings: + print(f" WARN: {warn}") + + return 0 if result.passed else 1 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 78185edc..7c7a7bfb 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,23 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard, _run_reviewer_briefing_check, _run_audit_chain_check, _run_pre_registration_check, _run_hypothesis_outcome_check, _run_baseline_comparison_check, _run_negative_result_check, _run_experiment_priority_check, _run_calibration_performance_check, _run_prediction_drift_check +from openamp_foundry.cli.commands.reports import ( + _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, + _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, + _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, + _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, + _run_synthetic_result_policy_check, _run_calibration_decision_checklist, + _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, + _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, + _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, + _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, + _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, + _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard, + _run_reviewer_briefing_check, _run_audit_chain_check, _run_pre_registration_check, + _run_hypothesis_outcome_check, _run_baseline_comparison_check, _run_negative_result_check, + _run_experiment_priority_check, _run_calibration_performance_check, _run_prediction_drift_check, + _run_calibration_improvement_check, +) from openamp_foundry.cli.commands.gates import _run_gate_check, _run_release_gate_check from openamp_foundry.cli.commands.reports import _run_contribution_check from openamp_foundry.cli.commands.reports import _run_decision_log @@ -2314,6 +2330,14 @@ def build_parser() -> argparse.ArgumentParser: p_drift.add_argument("--format", choices=["text", "json"], default="text") p_drift.set_defaults(func=_run_prediction_drift_check) + p_cir = sub.add_parser( + "calibration-improvement-check", + help="Validate calibration improvement record entry", + ) + p_cir.add_argument("--entry-json", default=None) + p_cir.add_argument("--format", choices=["text", "json"], default="text") + p_cir.set_defaults(func=_run_calibration_improvement_check) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", diff --git a/src/openamp_foundry/evidence/calibration_improvement_record.py b/src/openamp_foundry/evidence/calibration_improvement_record.py new file mode 100644 index 00000000..abd90bd5 --- /dev/null +++ b/src/openamp_foundry/evidence/calibration_improvement_record.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List, Set + +ACTION_TAKEN_MAX_LENGTH: int = 500 +VALID_ACTION_CATEGORIES: Set[str] = { + "threshold_adjustment", + "retraining", + "feature_removal", + "feature_addition", + "data_augmentation", + "weighting_change", +} +VALID_TRIGGER_PREFIXES: Set[str] = {"CPS-", "DRM-"} +POOR_BRIER_THRESHOLD: float = 0.25 + + +@dataclass +class CalibrationImprovementEntry: + improvement_id: str + pipeline_version_before: str + pipeline_version_after: str + trigger_ids: List[str] + action_taken: str + action_category: str + brier_score_before: float + brier_score_after: float + improvement_confirmed: bool + reviewer: str + dry_lab_only: bool = False + + +@dataclass +class CalibrationImprovementResult: + improvement_id: str + pipeline_version_before: str + pipeline_version_after: str + brier_score_before: float + brier_score_after: float + improvement_confirmed: bool + passed: bool + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + dry_lab_only: bool = False + + +def validate_calibration_improvement( + entry: CalibrationImprovementEntry, +) -> CalibrationImprovementResult: + errors: List[str] = [] + warnings: List[str] = [] + + if not entry.improvement_id.startswith("CIR-"): + errors.append( + f"improvement_id must start with 'CIR-', got '{entry.improvement_id}'" + ) + + if entry.pipeline_version_before == entry.pipeline_version_after: + errors.append( + "pipeline_version_before and pipeline_version_after must differ" + ) + + if not entry.trigger_ids: + errors.append("trigger_ids must contain at least 1 entry") + else: + bad_triggers = [ + tid for tid in entry.trigger_ids + if not any(tid.startswith(p) for p in VALID_TRIGGER_PREFIXES) + ] + if bad_triggers: + errors.append( + f"trigger_ids must start with CPS- or DRM-: {bad_triggers}" + ) + + if len(entry.action_taken) > ACTION_TAKEN_MAX_LENGTH: + errors.append( + f"action_taken exceeds {ACTION_TAKEN_MAX_LENGTH} chars " + f"(got {len(entry.action_taken)})" + ) + + if entry.action_category not in VALID_ACTION_CATEGORIES: + errors.append( + f"action_category must be one of {sorted(VALID_ACTION_CATEGORIES)}, " + f"got '{entry.action_category}'" + ) + + if not (0.0 <= entry.brier_score_before <= 1.0): + errors.append( + f"brier_score_before must be in [0.0, 1.0], " + f"got {entry.brier_score_before}" + ) + + if not (0.0 <= entry.brier_score_after <= 1.0): + errors.append( + f"brier_score_after must be in [0.0, 1.0], " + f"got {entry.brier_score_after}" + ) + + expected_confirmed = entry.brier_score_after < entry.brier_score_before + if entry.improvement_confirmed != expected_confirmed: + errors.append( + f"improvement_confirmed ({entry.improvement_confirmed}) is inconsistent: " + f"brier_score_after ({entry.brier_score_after}) " + f"{'<' if expected_confirmed else '>='} " + f"brier_score_before ({entry.brier_score_before})" + ) + + # Warnings + if entry.brier_score_after >= POOR_BRIER_THRESHOLD: + warnings.append( + f"brier_score_after ({entry.brier_score_after}) >= " + f"{POOR_BRIER_THRESHOLD} — calibration still poor after action" + ) + + return CalibrationImprovementResult( + improvement_id=entry.improvement_id, + pipeline_version_before=entry.pipeline_version_before, + pipeline_version_after=entry.pipeline_version_after, + brier_score_before=entry.brier_score_before, + brier_score_after=entry.brier_score_after, + improvement_confirmed=entry.improvement_confirmed, + passed=len(errors) == 0, + errors=errors, + warnings=warnings, + dry_lab_only=entry.dry_lab_only, + ) + + +def validate_calibration_improvement_dict(data: dict) -> CalibrationImprovementResult: + required = [ + "improvement_id", "pipeline_version_before", "pipeline_version_after", + "trigger_ids", "action_taken", "action_category", + "brier_score_before", "brier_score_after", "improvement_confirmed", "reviewer", + ] + errors: List[str] = [] + for key in required: + if key not in data: + errors.append(f"Missing required field: {key}") + if errors: + return CalibrationImprovementResult( + improvement_id=data.get("improvement_id", ""), + pipeline_version_before=data.get("pipeline_version_before", ""), + pipeline_version_after=data.get("pipeline_version_after", ""), + brier_score_before=float(data.get("brier_score_before", 0.0)), + brier_score_after=float(data.get("brier_score_after", 0.0)), + improvement_confirmed=bool(data.get("improvement_confirmed", False)), + passed=False, + errors=errors, + warnings=[], + ) + entry = CalibrationImprovementEntry( + improvement_id=str(data["improvement_id"]), + pipeline_version_before=str(data["pipeline_version_before"]), + pipeline_version_after=str(data["pipeline_version_after"]), + trigger_ids=list(data["trigger_ids"]), + action_taken=str(data["action_taken"]), + action_category=str(data["action_category"]), + brier_score_before=float(data["brier_score_before"]), + brier_score_after=float(data["brier_score_after"]), + improvement_confirmed=bool(data["improvement_confirmed"]), + reviewer=str(data["reviewer"]), + dry_lab_only=bool(data.get("dry_lab_only", False)), + ) + return validate_calibration_improvement(entry) diff --git a/tests/evidence/test_calibration_improvement_record.py b/tests/evidence/test_calibration_improvement_record.py new file mode 100644 index 00000000..4f2abf4c --- /dev/null +++ b/tests/evidence/test_calibration_improvement_record.py @@ -0,0 +1,377 @@ +"""Tests for calibration improvement record schema (Phase O O3).""" +import pytest +from openamp_foundry.evidence.calibration_improvement_record import ( + ACTION_TAKEN_MAX_LENGTH, + POOR_BRIER_THRESHOLD, + VALID_ACTION_CATEGORIES, + VALID_TRIGGER_PREFIXES, + CalibrationImprovementEntry, + CalibrationImprovementResult, + validate_calibration_improvement, + validate_calibration_improvement_dict, +) + + +def make_entry(**kwargs) -> CalibrationImprovementEntry: + defaults = dict( + improvement_id="CIR-001", + pipeline_version_before="v1.0.0", + pipeline_version_after="v1.1.0", + trigger_ids=["CPS-001"], + action_taken="Adjusted classification threshold from 0.5 to 0.45.", + action_category="threshold_adjustment", + brier_score_before=0.3, + brier_score_after=0.2, + improvement_confirmed=True, + reviewer="alice", + dry_lab_only=False, + ) + defaults.update(kwargs) + return CalibrationImprovementEntry(**defaults) + + +class TestConstants: + def test_action_taken_max_length(self): + assert ACTION_TAKEN_MAX_LENGTH == 500 + + def test_poor_brier_threshold(self): + assert POOR_BRIER_THRESHOLD == 0.25 + + def test_valid_action_categories_count(self): + assert len(VALID_ACTION_CATEGORIES) == 6 + + def test_valid_action_categories_contents(self): + assert "threshold_adjustment" in VALID_ACTION_CATEGORIES + assert "retraining" in VALID_ACTION_CATEGORIES + assert "feature_removal" in VALID_ACTION_CATEGORIES + assert "feature_addition" in VALID_ACTION_CATEGORIES + assert "data_augmentation" in VALID_ACTION_CATEGORIES + assert "weighting_change" in VALID_ACTION_CATEGORIES + + def test_valid_trigger_prefixes(self): + assert "CPS-" in VALID_TRIGGER_PREFIXES + assert "DRM-" in VALID_TRIGGER_PREFIXES + + +class TestHappyPath: + def test_valid_entry_passes(self): + entry = make_entry() + result = validate_calibration_improvement(entry) + assert result.passed + assert result.errors == [] + + def test_result_fields(self): + entry = make_entry() + result = validate_calibration_improvement(entry) + assert result.improvement_id == "CIR-001" + assert result.pipeline_version_before == "v1.0.0" + assert result.pipeline_version_after == "v1.1.0" + assert result.brier_score_before == 0.3 + assert result.brier_score_after == 0.2 + assert result.improvement_confirmed is True + assert result.dry_lab_only is False + + def test_dry_lab_only_default_false(self): + entry = CalibrationImprovementEntry( + improvement_id="CIR-001", + pipeline_version_before="v1.0", + pipeline_version_after="v1.1", + trigger_ids=["CPS-001"], + action_taken="threshold adjustment", + action_category="threshold_adjustment", + brier_score_before=0.3, + brier_score_after=0.2, + improvement_confirmed=True, + reviewer="bob", + ) + assert entry.dry_lab_only is False + + def test_no_improvement_confirmed_false(self): + entry = make_entry( + brier_score_before=0.2, + brier_score_after=0.3, + improvement_confirmed=False, + ) + result = validate_calibration_improvement(entry) + assert result.passed + assert result.errors == [] + + def test_equal_scores_confirmed_false(self): + entry = make_entry( + brier_score_before=0.2, + brier_score_after=0.2, + improvement_confirmed=False, + ) + result = validate_calibration_improvement(entry) + assert result.passed + + def test_multiple_trigger_ids(self): + entry = make_entry(trigger_ids=["CPS-001", "DRM-001", "CPS-002"]) + result = validate_calibration_improvement(entry) + assert result.passed + + def test_drm_trigger_id(self): + entry = make_entry(trigger_ids=["DRM-001"]) + result = validate_calibration_improvement(entry) + assert result.passed + + def test_all_action_categories(self): + for cat in VALID_ACTION_CATEGORIES: + entry = make_entry(action_category=cat) + result = validate_calibration_improvement(entry) + assert result.passed, f"Category {cat} should pass" + + +class TestImprovementIdValidation: + def test_wrong_prefix_fails(self): + entry = make_entry(improvement_id="WRONG-001") + result = validate_calibration_improvement(entry) + assert not result.passed + assert any("CIR-" in e for e in result.errors) + + def test_empty_id_fails(self): + entry = make_entry(improvement_id="") + result = validate_calibration_improvement(entry) + assert not result.passed + + def test_correct_prefix_passes(self): + entry = make_entry(improvement_id="CIR-XYZ-2026") + result = validate_calibration_improvement(entry) + assert result.passed + + +class TestVersionValidation: + def test_same_versions_fail(self): + entry = make_entry( + pipeline_version_before="v1.0", + pipeline_version_after="v1.0", + ) + result = validate_calibration_improvement(entry) + assert not result.passed + assert any("differ" in e for e in result.errors) + + def test_different_versions_pass(self): + entry = make_entry( + pipeline_version_before="v1.0", + pipeline_version_after="v1.1", + ) + result = validate_calibration_improvement(entry) + assert result.passed + + +class TestTriggerIdsValidation: + def test_empty_trigger_ids_fail(self): + entry = make_entry(trigger_ids=[]) + result = validate_calibration_improvement(entry) + assert not result.passed + assert any("trigger_ids" in e for e in result.errors) + + def test_invalid_prefix_trigger_fails(self): + entry = make_entry(trigger_ids=["WRONG-001"]) + result = validate_calibration_improvement(entry) + assert not result.passed + assert any("CPS-" in e or "DRM-" in e for e in result.errors) + + def test_mixed_valid_invalid_trigger_fails(self): + entry = make_entry(trigger_ids=["CPS-001", "BAD-001"]) + result = validate_calibration_improvement(entry) + assert not result.passed + + def test_valid_cps_trigger_passes(self): + entry = make_entry(trigger_ids=["CPS-XYZ"]) + result = validate_calibration_improvement(entry) + assert result.passed + + def test_valid_drm_trigger_passes(self): + entry = make_entry(trigger_ids=["DRM-XYZ"]) + result = validate_calibration_improvement(entry) + assert result.passed + + +class TestActionTakenValidation: + def test_too_long_action_fails(self): + entry = make_entry(action_taken="x" * 501) + result = validate_calibration_improvement(entry) + assert not result.passed + assert any("action_taken" in e for e in result.errors) + + def test_exact_max_length_passes(self): + entry = make_entry(action_taken="x" * 500) + result = validate_calibration_improvement(entry) + assert result.passed + + def test_empty_action_passes(self): + entry = make_entry(action_taken="") + result = validate_calibration_improvement(entry) + assert result.passed + + +class TestActionCategoryValidation: + def test_invalid_category_fails(self): + entry = make_entry(action_category="magic_fix") + result = validate_calibration_improvement(entry) + assert not result.passed + assert any("action_category" in e for e in result.errors) + + def test_empty_category_fails(self): + entry = make_entry(action_category="") + result = validate_calibration_improvement(entry) + assert not result.passed + + +class TestBrierScoreValidation: + def test_brier_before_below_zero_fails(self): + entry = make_entry(brier_score_before=-0.1, improvement_confirmed=False) + result = validate_calibration_improvement(entry) + assert not result.passed + assert any("brier_score_before" in e for e in result.errors) + + def test_brier_before_above_one_fails(self): + entry = make_entry(brier_score_before=1.1) + result = validate_calibration_improvement(entry) + assert not result.passed + + def test_brier_after_below_zero_fails(self): + entry = make_entry( + brier_score_after=-0.1, + improvement_confirmed=False, + ) + result = validate_calibration_improvement(entry) + assert not result.passed + assert any("brier_score_after" in e for e in result.errors) + + def test_brier_after_above_one_fails(self): + entry = make_entry(brier_score_after=1.1, improvement_confirmed=False) + result = validate_calibration_improvement(entry) + assert not result.passed + + def test_brier_zero_values_pass(self): + entry = make_entry( + brier_score_before=0.1, + brier_score_after=0.0, + improvement_confirmed=True, + ) + result = validate_calibration_improvement(entry) + assert result.passed + + +class TestImprovementConfirmedValidation: + def test_confirmed_true_but_no_improvement_fails(self): + entry = make_entry( + brier_score_before=0.2, + brier_score_after=0.3, + improvement_confirmed=True, + ) + result = validate_calibration_improvement(entry) + assert not result.passed + assert any("improvement_confirmed" in e for e in result.errors) + + def test_confirmed_false_but_improved_fails(self): + entry = make_entry( + brier_score_before=0.3, + brier_score_after=0.2, + improvement_confirmed=False, + ) + result = validate_calibration_improvement(entry) + assert not result.passed + assert any("improvement_confirmed" in e for e in result.errors) + + def test_confirmed_false_equal_scores_passes(self): + entry = make_entry( + brier_score_before=0.2, + brier_score_after=0.2, + improvement_confirmed=False, + ) + result = validate_calibration_improvement(entry) + assert result.passed + + def test_confirmed_true_improved_passes(self): + entry = make_entry( + brier_score_before=0.3, + brier_score_after=0.1, + improvement_confirmed=True, + ) + result = validate_calibration_improvement(entry) + assert result.passed + + +class TestWarnings: + def test_poor_brier_after_warns(self): + entry = make_entry( + brier_score_before=0.4, + brier_score_after=0.3, + improvement_confirmed=True, + ) + result = validate_calibration_improvement(entry) + assert result.passed + assert any("calibration still poor" in w for w in result.warnings) + + def test_good_brier_after_no_warn(self): + entry = make_entry( + brier_score_before=0.3, + brier_score_after=0.1, + improvement_confirmed=True, + ) + result = validate_calibration_improvement(entry) + assert result.passed + assert not any("calibration still poor" in w for w in result.warnings) + + def test_brier_at_threshold_warns(self): + entry = make_entry( + brier_score_before=0.4, + brier_score_after=0.25, + improvement_confirmed=True, + ) + result = validate_calibration_improvement(entry) + assert result.passed + assert any("calibration still poor" in w for w in result.warnings) + + def test_brier_just_below_threshold_no_warn(self): + entry = make_entry( + brier_score_before=0.3, + brier_score_after=0.249, + improvement_confirmed=True, + ) + result = validate_calibration_improvement(entry) + assert result.passed + assert not any("calibration still poor" in w for w in result.warnings) + + +class TestDictInterface: + def test_valid_dict_passes(self): + data = { + "improvement_id": "CIR-001", + "pipeline_version_before": "v1.0", + "pipeline_version_after": "v1.1", + "trigger_ids": ["CPS-001"], + "action_taken": "Adjusted threshold.", + "action_category": "threshold_adjustment", + "brier_score_before": 0.3, + "brier_score_after": 0.2, + "improvement_confirmed": True, + "reviewer": "alice", + } + result = validate_calibration_improvement_dict(data) + assert result.passed + + def test_missing_field_fails(self): + data = {"improvement_id": "CIR-001"} + result = validate_calibration_improvement_dict(data) + assert not result.passed + assert any("Missing required field" in e for e in result.errors) + + def test_dict_dry_lab_only_default_false(self): + data = { + "improvement_id": "CIR-001", + "pipeline_version_before": "v1.0", + "pipeline_version_after": "v1.1", + "trigger_ids": ["CPS-001"], + "action_taken": "threshold adjustment", + "action_category": "threshold_adjustment", + "brier_score_before": 0.3, + "brier_score_after": 0.2, + "improvement_confirmed": True, + "reviewer": "alice", + } + result = validate_calibration_improvement_dict(data) + assert result.dry_lab_only is False diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index ad823e3c..bd56e733 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 4617 +BASELINE = 4685 TOLERANCE = 0.05 # allow 5% fluctuation