From a5db14ed7e0e930019f58cc887e42fc3c27ba7a3 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 12:20:12 +0700 Subject: [PATCH] feat: Phase H H6 simulation uncertainty calibration report -- SUC- schema tracks whether module confidence intervals are empirically calibrated, overconfidence flag, 63 tests (#923) --- .../uncertainty_calibration_report.py | 170 +++++++ .../test_uncertainty_calibration_report.py | 420 ++++++++++++++++++ tests/test_test_count_regression.py | 2 +- 3 files changed, 591 insertions(+), 1 deletion(-) create mode 100644 src/openamp_foundry/simulation/uncertainty_calibration_report.py create mode 100644 tests/simulation/test_uncertainty_calibration_report.py diff --git a/src/openamp_foundry/simulation/uncertainty_calibration_report.py b/src/openamp_foundry/simulation/uncertainty_calibration_report.py new file mode 100644 index 00000000..1825c248 --- /dev/null +++ b/src/openamp_foundry/simulation/uncertainty_calibration_report.py @@ -0,0 +1,170 @@ +"""Simulation uncertainty calibration report schema (Phase H H6). + +SUC- records track whether a simulation module's uncertainty estimates +are well-calibrated — i.e. whether a stated 80% confidence interval +actually contains the true value 80% of the time. + +An over-confident module (claims narrow intervals but has low coverage) +is dangerous: it inflates trust in rankings without evidence. +An under-confident module wastes selection budget. +""" + +from dataclasses import dataclass, field + +SIMULATION_UNCERTAINTY_CALIBRATION_REPORT_ID_PREFIX = "SUC-" + +VALID_SIMULATION_MODULES: frozenset = frozenset({ + "membrane_proxy", + "structure_proxy", + "ensemble_checker", + "external_adapter", + "dummy_module", + "baseline_registry", + "other", +}) + +VALID_CALIBRATION_METHODS: frozenset = frozenset({ + "isotonic_regression", + "platt_scaling", + "temperature_scaling", + "histogram_binning", + "empirical_coverage_only", + "none_applied", +}) + +VALID_UNCERTAINTY_ASSESSMENT_STATUSES: frozenset = frozenset({ + "well_calibrated", + "overconfident", + "underconfident", + "insufficient_data", + "not_assessed", +}) + +OVERCONFIDENCE_THRESHOLD: float = 0.15 +UNDERCONFIDENCE_THRESHOLD: float = 0.15 +MIN_SAMPLES_FOR_CALIBRATION: int = 10 +CALIBRATION_ERROR_TOLERANCE: float = 0.01 + + +@dataclass +class SimulationUncertaintyCalibrationReport: + report_id: str + module_id: str + simulation_module: str + calibration_method: str + n_samples: int + expected_coverage: float + empirical_coverage: float + calibration_error: float + uncertainty_assessment_status: str + overconfidence_flag: bool + dry_lab_only: bool + cheap_baseline_beats_module: bool | None + notes: str + created_at: str + + +def validate_uncertainty_calibration_report( + report: SimulationUncertaintyCalibrationReport, +) -> list[str]: + errors: list[str] = [] + + if not report.report_id.startswith(SIMULATION_UNCERTAINTY_CALIBRATION_REPORT_ID_PREFIX): + errors.append( + f"report_id must start with '{SIMULATION_UNCERTAINTY_CALIBRATION_REPORT_ID_PREFIX}', " + f"got: {report.report_id!r}" + ) + + if report.simulation_module not in VALID_SIMULATION_MODULES: + errors.append( + f"simulation_module {report.simulation_module!r} not in VALID_SIMULATION_MODULES" + ) + + if report.calibration_method not in VALID_CALIBRATION_METHODS: + errors.append( + f"calibration_method {report.calibration_method!r} not in VALID_CALIBRATION_METHODS" + ) + + if report.uncertainty_assessment_status not in VALID_UNCERTAINTY_ASSESSMENT_STATUSES: + errors.append( + f"uncertainty_assessment_status {report.uncertainty_assessment_status!r} " + f"not in VALID_UNCERTAINTY_ASSESSMENT_STATUSES" + ) + + if not (0.0 <= report.expected_coverage <= 1.0): + errors.append( + f"expected_coverage must be in [0.0, 1.0], got: {report.expected_coverage}" + ) + + if not (0.0 <= report.empirical_coverage <= 1.0): + errors.append( + f"empirical_coverage must be in [0.0, 1.0], got: {report.empirical_coverage}" + ) + + computed_error = abs(report.expected_coverage - report.empirical_coverage) + if abs(computed_error - report.calibration_error) > CALIBRATION_ERROR_TOLERANCE: + errors.append( + f"calibration_error {report.calibration_error} does not match " + f"abs(expected_coverage - empirical_coverage) = {computed_error:.4f} " + f"(tolerance {CALIBRATION_ERROR_TOLERANCE})" + ) + + if ( + report.n_samples < MIN_SAMPLES_FOR_CALIBRATION + and report.uncertainty_assessment_status != "insufficient_data" + ): + errors.append( + f"n_samples={report.n_samples} < MIN_SAMPLES_FOR_CALIBRATION={MIN_SAMPLES_FOR_CALIBRATION}; " + f"uncertainty_assessment_status must be 'insufficient_data', " + f"got: {report.uncertainty_assessment_status!r}" + ) + + if not report.dry_lab_only: + errors.append("dry_lab_only must be True for all simulation calibration reports") + + if ( + report.uncertainty_assessment_status != "insufficient_data" + and report.empirical_coverage < report.expected_coverage - OVERCONFIDENCE_THRESHOLD + and not report.overconfidence_flag + ): + errors.append( + f"overconfidence_flag must be True when empirical_coverage " + f"({report.empirical_coverage}) < expected_coverage ({report.expected_coverage}) " + f"- OVERCONFIDENCE_THRESHOLD ({OVERCONFIDENCE_THRESHOLD})" + ) + + if not report.module_id.strip(): + errors.append("module_id must not be blank") + + if not report.created_at.strip(): + errors.append("created_at must not be blank") + + return errors + + +def format_uncertainty_calibration_report( + report: SimulationUncertaintyCalibrationReport, +) -> str: + lines = [ + f"Simulation Uncertainty Calibration Report {report.report_id}", + f"Module: {report.module_id} ({report.simulation_module})", + f"Calibration Method: {report.calibration_method}", + f"Samples: {report.n_samples}", + f"Expected Coverage: {report.expected_coverage:.1%}", + f"Empirical Coverage: {report.empirical_coverage:.1%}", + f"Calibration Error: {report.calibration_error:.4f}", + f"Assessment: {report.uncertainty_assessment_status}", + ] + if report.overconfidence_flag: + lines.append("WARNING: Module is over-confident (narrow intervals, low coverage)") + if report.cheap_baseline_beats_module is True: + lines.append("NOTE: Cheap baseline beats this module — consider deprecation") + elif report.cheap_baseline_beats_module is False: + lines.append("Cheap baseline does NOT beat this module") + else: + lines.append("Cheap baseline comparison: not yet assessed") + lines.append(f"Dry-lab only: {report.dry_lab_only}") + if report.notes: + lines.append(f"Notes: {report.notes}") + lines.append(f"Created: {report.created_at}") + return "\n".join(lines) diff --git a/tests/simulation/test_uncertainty_calibration_report.py b/tests/simulation/test_uncertainty_calibration_report.py new file mode 100644 index 00000000..1b3b133c --- /dev/null +++ b/tests/simulation/test_uncertainty_calibration_report.py @@ -0,0 +1,420 @@ +"""Tests for simulation uncertainty calibration report schema (Phase H H6). + +5 test classes, 63 tests total. +""" + +import pytest + +from openamp_foundry.simulation.uncertainty_calibration_report import ( + CALIBRATION_ERROR_TOLERANCE, + MIN_SAMPLES_FOR_CALIBRATION, + OVERCONFIDENCE_THRESHOLD, + SIMULATION_UNCERTAINTY_CALIBRATION_REPORT_ID_PREFIX, + UNDERCONFIDENCE_THRESHOLD, + VALID_CALIBRATION_METHODS, + VALID_SIMULATION_MODULES, + VALID_UNCERTAINTY_ASSESSMENT_STATUSES, + SimulationUncertaintyCalibrationReport, + format_uncertainty_calibration_report, + validate_uncertainty_calibration_report, +) + + +def _valid_report(**kwargs) -> SimulationUncertaintyCalibrationReport: + defaults = dict( + report_id="SUC-001", + module_id="MEM-001", + simulation_module="membrane_proxy", + calibration_method="empirical_coverage_only", + n_samples=50, + expected_coverage=0.80, + empirical_coverage=0.82, + calibration_error=0.02, + uncertainty_assessment_status="well_calibrated", + overconfidence_flag=False, + dry_lab_only=True, + cheap_baseline_beats_module=None, + notes="Routine calibration check", + created_at="2026-01-01", + ) + defaults.update(kwargs) + return SimulationUncertaintyCalibrationReport(**defaults) + + +# --------------------------------------------------------------------------- +# Class 1: Constants (10 tests) +# --------------------------------------------------------------------------- + +class TestSimulationUncertaintyCalibrationReportConstants: + def test_prefix_value(self): + assert SIMULATION_UNCERTAINTY_CALIBRATION_REPORT_ID_PREFIX == "SUC-" + + def test_overconfidence_threshold_positive(self): + assert OVERCONFIDENCE_THRESHOLD > 0.0 + + def test_underconfidence_threshold_positive(self): + assert UNDERCONFIDENCE_THRESHOLD > 0.0 + + def test_min_samples_positive(self): + assert MIN_SAMPLES_FOR_CALIBRATION > 0 + + def test_calibration_error_tolerance_small(self): + assert CALIBRATION_ERROR_TOLERANCE < 0.1 + + def test_valid_simulation_modules_nonempty(self): + assert len(VALID_SIMULATION_MODULES) >= 5 + + def test_valid_calibration_methods_nonempty(self): + assert len(VALID_CALIBRATION_METHODS) >= 4 + + def test_valid_assessment_statuses_nonempty(self): + assert len(VALID_UNCERTAINTY_ASSESSMENT_STATUSES) >= 4 + + def test_membrane_proxy_in_modules(self): + assert "membrane_proxy" in VALID_SIMULATION_MODULES + + def test_none_applied_in_calibration_methods(self): + assert "none_applied" in VALID_CALIBRATION_METHODS + + +# --------------------------------------------------------------------------- +# Class 2: Validation happy and sad paths (20 tests) +# --------------------------------------------------------------------------- + +class TestValidateSimulationUncertaintyCalibrationReport: + def test_valid_report_has_no_errors(self): + r = _valid_report() + assert validate_uncertainty_calibration_report(r) == [] + + def test_bad_prefix_rejected(self): + r = _valid_report(report_id="BAD-001") + errs = validate_uncertainty_calibration_report(r) + assert any("report_id" in e for e in errs) + + def test_invalid_simulation_module_rejected(self): + r = _valid_report(simulation_module="unknown_module") + errs = validate_uncertainty_calibration_report(r) + assert any("simulation_module" in e for e in errs) + + def test_invalid_calibration_method_rejected(self): + r = _valid_report(calibration_method="magic_method") + errs = validate_uncertainty_calibration_report(r) + assert any("calibration_method" in e for e in errs) + + def test_invalid_assessment_status_rejected(self): + r = _valid_report(uncertainty_assessment_status="great") + errs = validate_uncertainty_calibration_report(r) + assert any("uncertainty_assessment_status" in e for e in errs) + + def test_expected_coverage_above_one_rejected(self): + r = _valid_report(expected_coverage=1.5, empirical_coverage=0.8, + calibration_error=0.7) + errs = validate_uncertainty_calibration_report(r) + assert any("expected_coverage" in e for e in errs) + + def test_expected_coverage_below_zero_rejected(self): + r = _valid_report(expected_coverage=-0.1, empirical_coverage=0.8, + calibration_error=0.9) + errs = validate_uncertainty_calibration_report(r) + assert any("expected_coverage" in e for e in errs) + + def test_empirical_coverage_above_one_rejected(self): + r = _valid_report(empirical_coverage=1.1, calibration_error=0.3) + errs = validate_uncertainty_calibration_report(r) + assert any("empirical_coverage" in e for e in errs) + + def test_calibration_error_mismatch_rejected(self): + r = _valid_report(expected_coverage=0.80, empirical_coverage=0.82, + calibration_error=0.99) + errs = validate_uncertainty_calibration_report(r) + assert any("calibration_error" in e for e in errs) + + def test_insufficient_samples_requires_status(self): + r = _valid_report( + n_samples=5, + uncertainty_assessment_status="well_calibrated", + ) + errs = validate_uncertainty_calibration_report(r) + assert any("insufficient_data" in e for e in errs) + + def test_insufficient_samples_with_correct_status_ok(self): + r = _valid_report( + n_samples=5, + uncertainty_assessment_status="insufficient_data", + ) + errs = validate_uncertainty_calibration_report(r) + assert not any("insufficient_data" in e for e in errs) + + def test_dry_lab_only_false_rejected(self): + r = _valid_report(dry_lab_only=False) + errs = validate_uncertainty_calibration_report(r) + assert any("dry_lab_only" in e for e in errs) + + def test_overconfidence_flag_must_be_true_when_coverage_low(self): + low_coverage = 0.80 - OVERCONFIDENCE_THRESHOLD - 0.05 + err = abs(0.80 - low_coverage) + r = _valid_report( + expected_coverage=0.80, + empirical_coverage=low_coverage, + calibration_error=round(err, 4), + uncertainty_assessment_status="overconfident", + overconfidence_flag=False, + ) + errs = validate_uncertainty_calibration_report(r) + assert any("overconfidence_flag" in e for e in errs) + + def test_overconfidence_flag_true_when_coverage_low_passes(self): + low_coverage = 0.80 - OVERCONFIDENCE_THRESHOLD - 0.05 + err = abs(0.80 - low_coverage) + r = _valid_report( + expected_coverage=0.80, + empirical_coverage=low_coverage, + calibration_error=round(err, 4), + uncertainty_assessment_status="overconfident", + overconfidence_flag=True, + ) + errs = validate_uncertainty_calibration_report(r) + assert not any("overconfidence_flag" in e for e in errs) + + def test_blank_module_id_rejected(self): + r = _valid_report(module_id=" ") + errs = validate_uncertainty_calibration_report(r) + assert any("module_id" in e for e in errs) + + def test_blank_created_at_rejected(self): + r = _valid_report(created_at="") + errs = validate_uncertainty_calibration_report(r) + assert any("created_at" in e for e in errs) + + def test_all_simulation_modules_valid(self): + for mod in VALID_SIMULATION_MODULES: + r = _valid_report(simulation_module=mod) + errs = validate_uncertainty_calibration_report(r) + assert not any("simulation_module" in e for e in errs), mod + + def test_all_calibration_methods_valid(self): + for method in VALID_CALIBRATION_METHODS: + r = _valid_report(calibration_method=method) + errs = validate_uncertainty_calibration_report(r) + assert not any("calibration_method" in e for e in errs), method + + def test_all_assessment_statuses_valid_with_sufficient_samples(self): + for status in VALID_UNCERTAINTY_ASSESSMENT_STATUSES: + if status == "insufficient_data": + continue + r = _valid_report(uncertainty_assessment_status=status) + errs = validate_uncertainty_calibration_report(r) + assert not any("uncertainty_assessment_status" in e for e in errs), status + + def test_calibration_error_within_tolerance_passes(self): + r = _valid_report( + expected_coverage=0.80, + empirical_coverage=0.805, + calibration_error=0.005, + ) + errs = validate_uncertainty_calibration_report(r) + assert not any("calibration_error" in e for e in errs) + + +# --------------------------------------------------------------------------- +# Class 3: Edge cases (13 tests) +# --------------------------------------------------------------------------- + +class TestSimulationUncertaintyCalibrationReportEdgeCases: + def test_zero_calibration_error_perfect_calibration(self): + r = _valid_report( + expected_coverage=0.80, + empirical_coverage=0.80, + calibration_error=0.0, + ) + assert validate_uncertainty_calibration_report(r) == [] + + def test_exactly_min_samples_ok(self): + r = _valid_report(n_samples=MIN_SAMPLES_FOR_CALIBRATION) + assert validate_uncertainty_calibration_report(r) == [] + + def test_one_below_min_samples_requires_insufficient(self): + r = _valid_report( + n_samples=MIN_SAMPLES_FOR_CALIBRATION - 1, + uncertainty_assessment_status="well_calibrated", + ) + errs = validate_uncertainty_calibration_report(r) + assert any("insufficient_data" in e for e in errs) + + def test_cheap_baseline_beats_module_none_ok(self): + r = _valid_report(cheap_baseline_beats_module=None) + assert validate_uncertainty_calibration_report(r) == [] + + def test_cheap_baseline_beats_module_true_ok(self): + r = _valid_report(cheap_baseline_beats_module=True) + assert validate_uncertainty_calibration_report(r) == [] + + def test_cheap_baseline_beats_module_false_ok(self): + r = _valid_report(cheap_baseline_beats_module=False) + assert validate_uncertainty_calibration_report(r) == [] + + def test_expected_coverage_zero_valid(self): + r = _valid_report( + expected_coverage=0.0, + empirical_coverage=0.0, + calibration_error=0.0, + ) + assert validate_uncertainty_calibration_report(r) == [] + + def test_expected_coverage_one_valid(self): + r = _valid_report( + expected_coverage=1.0, + empirical_coverage=1.0, + calibration_error=0.0, + ) + assert validate_uncertainty_calibration_report(r) == [] + + def test_not_assessed_status_ok_with_sufficient_samples(self): + r = _valid_report(uncertainty_assessment_status="not_assessed") + assert validate_uncertainty_calibration_report(r) == [] + + def test_underconfident_status_ok_when_coverage_high(self): + r = _valid_report( + expected_coverage=0.70, + empirical_coverage=0.95, + calibration_error=0.25, + uncertainty_assessment_status="underconfident", + overconfidence_flag=False, + ) + assert validate_uncertainty_calibration_report(r) == [] + + def test_empty_notes_ok(self): + r = _valid_report(notes="") + assert validate_uncertainty_calibration_report(r) == [] + + def test_overconfidence_exactly_at_threshold_no_flag_required(self): + threshold_coverage = 0.80 - OVERCONFIDENCE_THRESHOLD + err = abs(0.80 - threshold_coverage) + r = _valid_report( + expected_coverage=0.80, + empirical_coverage=threshold_coverage, + calibration_error=round(err, 4), + uncertainty_assessment_status="overconfident", + overconfidence_flag=False, + ) + errs = validate_uncertainty_calibration_report(r) + assert not any("overconfidence_flag" in e for e in errs) + + def test_prefix_case_sensitive(self): + r = _valid_report(report_id="suc-001") + errs = validate_uncertainty_calibration_report(r) + assert any("report_id" in e for e in errs) + + +# --------------------------------------------------------------------------- +# Class 4: Format output (10 tests) +# --------------------------------------------------------------------------- + +class TestFormatSimulationUncertaintyCalibrationReport: + def test_format_contains_report_id(self): + r = _valid_report() + out = format_uncertainty_calibration_report(r) + assert "SUC-001" in out + + def test_format_contains_module_id(self): + r = _valid_report() + out = format_uncertainty_calibration_report(r) + assert "MEM-001" in out + + def test_format_contains_expected_coverage(self): + r = _valid_report() + out = format_uncertainty_calibration_report(r) + assert "80.0%" in out or "80%" in out + + def test_format_contains_empirical_coverage(self): + r = _valid_report() + out = format_uncertainty_calibration_report(r) + assert "82.0%" in out or "82%" in out + + def test_format_contains_assessment_status(self): + r = _valid_report() + out = format_uncertainty_calibration_report(r) + assert "well_calibrated" in out + + def test_format_overconfidence_warning_when_flag_true(self): + r = _valid_report(overconfidence_flag=True) + out = format_uncertainty_calibration_report(r) + assert "over-confident" in out.lower() or "overconfident" in out.lower() or "WARNING" in out + + def test_format_baseline_beats_note_when_true(self): + r = _valid_report(cheap_baseline_beats_module=True) + out = format_uncertainty_calibration_report(r) + assert "baseline" in out.lower() + + def test_format_baseline_none_note(self): + r = _valid_report(cheap_baseline_beats_module=None) + out = format_uncertainty_calibration_report(r) + assert "not yet assessed" in out.lower() or "none" in out.lower() or "baseline" in out.lower() + + def test_format_contains_dry_lab_only(self): + r = _valid_report() + out = format_uncertainty_calibration_report(r) + assert "True" in out + + def test_format_is_string(self): + r = _valid_report() + out = format_uncertainty_calibration_report(r) + assert isinstance(out, str) + assert len(out) > 50 + + +# --------------------------------------------------------------------------- +# Class 5: Dry-lab enforcement (10 tests) +# --------------------------------------------------------------------------- + +class TestSimulationUncertaintyCalibrationReportDryLabEnforcement: + def test_dry_lab_only_true_required(self): + r = _valid_report(dry_lab_only=False) + errs = validate_uncertainty_calibration_report(r) + assert len(errs) >= 1 + + def test_dry_lab_only_true_passes(self): + r = _valid_report(dry_lab_only=True) + assert validate_uncertainty_calibration_report(r) == [] + + def test_error_message_mentions_dry_lab(self): + r = _valid_report(dry_lab_only=False) + errs = validate_uncertainty_calibration_report(r) + assert any("dry_lab" in e for e in errs) + + def test_all_modules_enforce_dry_lab(self): + for mod in list(VALID_SIMULATION_MODULES)[:3]: + r = _valid_report(simulation_module=mod, dry_lab_only=False) + errs = validate_uncertainty_calibration_report(r) + assert any("dry_lab" in e for e in errs), mod + + def test_format_shows_dry_lab_true(self): + r = _valid_report(dry_lab_only=True) + out = format_uncertainty_calibration_report(r) + assert "True" in out + + def test_suc_prefix_is_mandatory(self): + r = _valid_report(report_id="UCR-001") + errs = validate_uncertainty_calibration_report(r) + assert any("SUC-" in e for e in errs) + + def test_well_calibrated_near_zero_error(self): + r = _valid_report( + expected_coverage=0.90, + empirical_coverage=0.91, + calibration_error=0.01, + uncertainty_assessment_status="well_calibrated", + ) + assert validate_uncertainty_calibration_report(r) == [] + + def test_n_samples_large_ok(self): + r = _valid_report(n_samples=1000) + assert validate_uncertainty_calibration_report(r) == [] + + def test_notes_with_special_chars_ok(self): + r = _valid_report(notes="Module tested on TOY-001 to TOY-005; all dry-lab only.") + assert validate_uncertainty_calibration_report(r) == [] + + def test_isotonic_regression_calibration_method_ok(self): + r = _valid_report(calibration_method="isotonic_regression") + assert validate_uncertainty_calibration_report(r) == [] diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index c05bb61c..02a1db8a 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -3,7 +3,7 @@ import subprocess import math -BASELINE = 8221 +BASELINE = 8284 def test_test_count_regression():