From ff82ef63a26a1d09b39b9e17a4d4f572f6e04e58 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 09:14:14 +0700 Subject: [PATCH] feat: Phase G G2 recalibration rejection summary schema -- RRS- aggregate refusal audit with rate consistency check and gate effectiveness tracking --- docs/evidence/METRICS_CURRENT.md | 12 +- .../RECALIBRATION_REJECTION_SUMMARY_GUIDE.md | 66 ++ docs/research/NEXT_100_PR_MAP.md | 2 +- src/openamp_foundry/cli/commands/reports.py | 17 + src/openamp_foundry/cli/main.py | 5 + .../recalibration_rejection_summary.py | 197 ++++++ .../test_recalibration_rejection_summary.py | 597 ++++++++++++++++++ tests/test_test_count_regression.py | 2 +- 8 files changed, 895 insertions(+), 3 deletions(-) create mode 100644 docs/evidence/RECALIBRATION_REJECTION_SUMMARY_GUIDE.md create mode 100644 src/openamp_foundry/evidence/recalibration_rejection_summary.py create mode 100644 tests/evidence/test_recalibration_rejection_summary.py diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index e2f91d53..9f50867a 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,17 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-10 (Phase G G1 — recalibration decision log — v0.10.21) +> **Last updated:** 2026-07-10 (Phase G G2 — recalibration rejection summary — v0.10.22) + +## Changelog + +### v0.10.22 — Phase G G2: RecalibrationRejectionSummary schema +- Added `RecalibrationRejectionSummary` (RRS-) schema +- 13 fields: rrs_id, pipeline_version, period_start, period_end, total_checkpoints_reviewed, total_refused, total_approved, refusal_rate, top_refusal_reason, gate_status, all_refusals_have_rrf, summary, notes +- 15 validation rules; 3 warnings; 63 tests +- BASELINE 5981→6044 +- CLI: openamp-foundry recalibration-rejection-summary-check +- Closes Phase G G2 — aggregate refusal audit trail showing gate effectiveness ## v0.10.20 — Phase F F9 — Negative Result Dashboard (NRD-) diff --git a/docs/evidence/RECALIBRATION_REJECTION_SUMMARY_GUIDE.md b/docs/evidence/RECALIBRATION_REJECTION_SUMMARY_GUIDE.md new file mode 100644 index 00000000..97df1366 --- /dev/null +++ b/docs/evidence/RECALIBRATION_REJECTION_SUMMARY_GUIDE.md @@ -0,0 +1,66 @@ +# RecalibrationRejectionSummary (RRS-) Guide + +## Purpose + +`RecalibrationRejectionSummary` aggregates recalibration refusal events across +a pipeline period to show that the calibration gate is working correctly — +that the right recalibrations were correctly refused, not rubber-stamped. + +## Schema Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `rrs_id` | str | yes | Unique identifier, must start with `RRS-` | +| `pipeline_version` | str | yes | Pipeline version during this period | +| `period_start` | str | yes | ISO date (YYYY-MM-DD) for period start | +| `period_end` | str | yes | ISO date (YYYY-MM-DD) for period end | +| `total_checkpoints_reviewed` | int | yes | Total calibration checkpoints reviewed (≥1) | +| `total_refused` | int | yes | Number refused (in [0, total_checkpoints_reviewed]) | +| `total_approved` | int | yes | Number approved (refused+approved=total) | +| `refusal_rate` | float | yes | Fraction refused (must match total_refused/total, tol 0.01) | +| `top_refusal_reason` | str | yes | Most common refusal reason (controlled vocabulary) | +| `gate_status` | str | yes | Current gate status (`active`, `suspended`, `under_review`) | +| `all_refusals_have_rrf` | bool | yes | Must be True — all refusals have RRF- artifacts | +| `summary` | str | yes | Narrative summary (≤400 chars) | +| `notes` | str | no | Additional context (≤300 chars) | + +## Controlled Vocabularies + +**top_refusal_reason (7 values):** +- `insufficient_data`, `effect_within_noise`, `recent_recalibration` +- `conflicting_signals`, `reviewer_override`, `multiple_reasons`, `none_refused` + +**gate_status (3 values):** +- `active` — gate operating normally +- `suspended` — gate temporarily suspended (requires human approval) +- `under_review` — gate configuration under review + +## Validation Rules + +1. `rrs_id` starts with `RRS-` +2. `pipeline_version` non-empty +3. `period_start` is ISO date (YYYY-MM-DD) +4. `period_end` is ISO date (YYYY-MM-DD) +5. `total_checkpoints_reviewed` ≥ 1 +6. `total_refused` in [0, total_checkpoints_reviewed] +7. `total_approved` in [0, total_checkpoints_reviewed] +8. `total_refused + total_approved == total_checkpoints_reviewed` +9. `refusal_rate` in [0.0, 1.0] +10. `refusal_rate` consistent with `total_refused / total_checkpoints_reviewed` (tolerance 0.01) +11. `top_refusal_reason` in controlled vocabulary (7 values) +12. `gate_status` in controlled vocabulary (3 values) +13. `all_refusals_have_rrf` must be True +14. `summary` non-empty and ≤400 chars +15. `notes` ≤300 chars + +## Warnings + +- `refusal_rate > 0.9`: investigate whether gate thresholds are too strict +- `total_refused == 0`: verify gate is operating (not silently approving all) +- `notes` empty: consider adding context for reviewers + +## Integration + +- Each refusal references: `RRF-` (recalibration refusal record) +- Triggered by: `RDL-` (recalibration decision log) showing refused outcomes +- Feeds into: governance audit trail, calibration health monitoring diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index a4f055f7..417be7af 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -127,7 +127,7 @@ Make learning controlled rather than self-serving. | PR | Task | Why it matters | Review class | |---:|---|---|---| | G1 | Add machine-readable recalibration decision log examples (complete). | RDL- schema: 13 fields, 13 validation rules, 4 outcomes, 5 trigger types, deferred-requires-conditions enforced; closes governance audit trail for calibration decisions. | B | -| G2 | Add recalibration rejection examples. | Shows gate success by refusal. | B | +| G2 | Add recalibration rejection examples (complete). | RRS- schema: 13 fields, 15 validation rules, refusal_rate consistency check (tol 0.01), all_refusals_have_rrf=True enforced, gate status controlled vocabulary; shows gate success by audit-ready refusal aggregation. | B | | G3 | Add `make calibration-audit`. | Checks intake/gate/engine/report consistency. | C | | G4 | Add active-learning strategy comparison report. | Prevents one-selector bias. | C | | G5 | Add batch-2 rationale report explaining exploit/explore/diversity roles. | Reviewers understand next batch. | C | diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index b0bec099..4ea2b66b 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1302,6 +1302,23 @@ def _run_recalibration_decision_log_check(args): print("OK: RecalibrationDecisionLog is valid.") +def _run_recalibration_rejection_summary_check(args): + """CLI handler for recalibration-rejection-summary-check.""" + import json, sys + from openamp_foundry.evidence.recalibration_rejection_summary import validate_dict + data = json.load(open(args.input)) + issues = validate_dict(data) + errors = [i for i in issues if not i.startswith("WARNING:")] + warnings = [i for i in issues if i.startswith("WARNING:")] + for w in warnings: + print(w) + if errors: + for e in errors: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + print("OK: RecalibrationRejectionSummary is valid.") + + def _run_domain_review_outcome_check(args): from openamp_foundry.evidence.domain_review_outcome import ( validate_domain_review_outcome_dict, diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index a9da318a..e31ba201 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -37,6 +37,7 @@ _run_pilot_evidence_package_check, _run_pre_registration_entry_check, _run_recalibration_decision_log_check, + _run_recalibration_rejection_summary_check, ) from openamp_foundry.cli.commands.gates import _run_gate_check, _run_release_gate_check from openamp_foundry.cli.commands.reports import _run_contribution_check @@ -2533,6 +2534,10 @@ def build_parser() -> argparse.ArgumentParser: p_rdl.add_argument("input", help="Path to JSON file") p_rdl.set_defaults(func=_run_recalibration_decision_log_check) + p_rrs = subparsers.add_parser("recalibration-rejection-summary-check", help="Validate a RecalibrationRejectionSummary JSON") + p_rrs.add_argument("input", help="Path to JSON file") + p_rrs.set_defaults(func=_run_recalibration_rejection_summary_check) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", diff --git a/src/openamp_foundry/evidence/recalibration_rejection_summary.py b/src/openamp_foundry/evidence/recalibration_rejection_summary.py new file mode 100644 index 00000000..5969c1f4 --- /dev/null +++ b/src/openamp_foundry/evidence/recalibration_rejection_summary.py @@ -0,0 +1,197 @@ +"""Recalibration rejection summary schema — Phase G G2. + +Aggregates multiple recalibration refusal events across a pipeline period +to show that the calibration gate is working correctly — that the right +recalibrations were correctly refused, not rubber-stamped. + +A healthy rejection summary shows refusals happening for documented reasons, +not silence or blanket approval. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List + +RRS_PREFIX = "RRS-" + +VALID_TOP_REFUSAL_REASONS = frozenset({ + "insufficient_data", + "effect_within_noise", + "recent_recalibration", + "conflicting_signals", + "reviewer_override", + "multiple_reasons", + "none_refused", +}) + +VALID_GATE_STATUSES = frozenset({ + "active", + "suspended", + "under_review", +}) + +REFUSAL_RATE_TOLERANCE = 0.01 +HIGH_REFUSAL_RATE_WARNING = 0.9 +ZERO_REFUSAL_WARNING_THRESHOLD = 0 # warn when zero refusals observed +SUMMARY_MAX_LENGTH = 400 +NOTES_MAX_LENGTH = 300 +_ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass +class RecalibrationRejectionSummary: + rrs_id: str + pipeline_version: str + period_start: str + period_end: str + total_checkpoints_reviewed: int + total_refused: int + total_approved: int + refusal_rate: float + top_refusal_reason: str + gate_status: str + all_refusals_have_rrf: bool + summary: str + notes: str = "" + + +def validate(summary: RecalibrationRejectionSummary) -> List[str]: + """Return list of error strings; empty list means valid.""" + errors: List[str] = [] + + # Rule 1: ID prefix + if not summary.rrs_id.startswith(RRS_PREFIX): + errors.append( + f"rrs_id must start with '{RRS_PREFIX}', got: {summary.rrs_id!r}" + ) + + # Rule 2: pipeline_version non-empty + if not summary.pipeline_version.strip(): + errors.append("pipeline_version must not be empty") + + # Rule 3: period_start ISO format + if not _ISO_DATE_RE.match(summary.period_start): + errors.append( + f"period_start must be YYYY-MM-DD, got: {summary.period_start!r}" + ) + + # Rule 4: period_end ISO format + if not _ISO_DATE_RE.match(summary.period_end): + errors.append( + f"period_end must be YYYY-MM-DD, got: {summary.period_end!r}" + ) + + # Rule 5: total_checkpoints_reviewed >= 1 + if summary.total_checkpoints_reviewed < 1: + errors.append( + f"total_checkpoints_reviewed must be >= 1, " + f"got: {summary.total_checkpoints_reviewed}" + ) + + # Rule 6: total_refused in [0, total_checkpoints_reviewed] + if not (0 <= summary.total_refused <= summary.total_checkpoints_reviewed): + errors.append( + f"total_refused must be in [0, {summary.total_checkpoints_reviewed}], " + f"got: {summary.total_refused}" + ) + + # Rule 7: total_approved in [0, total_checkpoints_reviewed] + if not (0 <= summary.total_approved <= summary.total_checkpoints_reviewed): + errors.append( + f"total_approved must be in [0, {summary.total_checkpoints_reviewed}], " + f"got: {summary.total_approved}" + ) + + # Rule 8: total_refused + total_approved == total_checkpoints_reviewed + if ( + summary.total_refused + summary.total_approved + != summary.total_checkpoints_reviewed + ): + errors.append( + f"total_refused ({summary.total_refused}) + " + f"total_approved ({summary.total_approved}) must equal " + f"total_checkpoints_reviewed ({summary.total_checkpoints_reviewed})" + ) + + # Rule 9: refusal_rate in [0.0, 1.0] + if not (0.0 <= summary.refusal_rate <= 1.0): + errors.append( + f"refusal_rate must be in [0.0, 1.0], got: {summary.refusal_rate}" + ) + + # Rule 10: refusal_rate consistency check (tolerance 0.01) + if summary.total_checkpoints_reviewed >= 1: + computed = summary.total_refused / summary.total_checkpoints_reviewed + if abs(computed - summary.refusal_rate) > REFUSAL_RATE_TOLERANCE: + errors.append( + f"refusal_rate ({summary.refusal_rate:.4f}) inconsistent with " + f"total_refused/total_checkpoints_reviewed " + f"({summary.total_refused}/{summary.total_checkpoints_reviewed} " + f"= {computed:.4f}); tolerance is {REFUSAL_RATE_TOLERANCE}" + ) + + # Rule 11: top_refusal_reason vocabulary + if summary.top_refusal_reason not in VALID_TOP_REFUSAL_REASONS: + errors.append( + f"top_refusal_reason must be one of {sorted(VALID_TOP_REFUSAL_REASONS)}, " + f"got: {summary.top_refusal_reason!r}" + ) + + # Rule 12: gate_status vocabulary + if summary.gate_status not in VALID_GATE_STATUSES: + errors.append( + f"gate_status must be one of {sorted(VALID_GATE_STATUSES)}, " + f"got: {summary.gate_status!r}" + ) + + # Rule 13: all_refusals_have_rrf must be True + if not summary.all_refusals_have_rrf: + errors.append( + "all_refusals_have_rrf must be True — every refusal must " + "have a corresponding RRF- artifact" + ) + + # Rule 14: summary non-empty and length + if not summary.summary.strip(): + errors.append("summary must not be empty") + elif len(summary.summary) > SUMMARY_MAX_LENGTH: + errors.append( + f"summary exceeds {SUMMARY_MAX_LENGTH} chars " + f"(got {len(summary.summary)})" + ) + + # Rule 15: notes length + if len(summary.notes) > NOTES_MAX_LENGTH: + errors.append( + f"notes exceeds {NOTES_MAX_LENGTH} chars (got {len(summary.notes)})" + ) + + # Warnings + if summary.refusal_rate >= HIGH_REFUSAL_RATE_WARNING: + errors.append( + f"WARNING: refusal_rate ({summary.refusal_rate:.2f}) > " + f"{HIGH_REFUSAL_RATE_WARNING} — investigate whether gate thresholds " + "are too strict" + ) + if summary.total_refused == ZERO_REFUSAL_WARNING_THRESHOLD: + errors.append( + "WARNING: total_refused is 0 — verify that the gate is operating " + "and not silently approving all recalibrations" + ) + if not summary.notes.strip(): + errors.append( + "WARNING: notes is empty — consider adding context for reviewers" + ) + + return errors + + +def validate_dict(data: dict) -> List[str]: + """Validate a plain dict by constructing RecalibrationRejectionSummary first.""" + try: + s = RecalibrationRejectionSummary(**data) + except TypeError as exc: + return [f"Schema construction error: {exc}"] + return validate(s) diff --git a/tests/evidence/test_recalibration_rejection_summary.py b/tests/evidence/test_recalibration_rejection_summary.py new file mode 100644 index 00000000..14cab43f --- /dev/null +++ b/tests/evidence/test_recalibration_rejection_summary.py @@ -0,0 +1,597 @@ +"""Tests for RecalibrationRejectionSummary schema — Phase G G2. + +Exactly 63 tests: valid baseline + each validation rule + edge cases + warnings. +""" + +from __future__ import annotations + +import pytest + +from openamp_foundry.evidence.recalibration_rejection_summary import ( + NOTES_MAX_LENGTH, + RRS_PREFIX, + REFUSAL_RATE_TOLERANCE, + SUMMARY_MAX_LENGTH, + VALID_GATE_STATUSES, + VALID_TOP_REFUSAL_REASONS, + RecalibrationRejectionSummary, + validate, + validate_dict, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _valid_summary(**overrides) -> RecalibrationRejectionSummary: + defaults = dict( + rrs_id="RRS-20240315-001", + pipeline_version="v2.1.0", + period_start="2024-01-01", + period_end="2024-03-31", + total_checkpoints_reviewed=10, + total_refused=3, + total_approved=7, + refusal_rate=0.30, + top_refusal_reason="effect_within_noise", + gate_status="active", + all_refusals_have_rrf=True, + summary="3 of 10 checkpoints refused: noise-within-tolerance (2), insufficient data (1).", + notes="", + ) + defaults.update(overrides) + return RecalibrationRejectionSummary(**defaults) + + +def _valid() -> RecalibrationRejectionSummary: + return _valid_summary() + + +def _errors(s): + return [e for e in validate(s) if not e.startswith("WARNING:")] + + +def _warns(s): + return [e for e in validate(s) if e.startswith("WARNING:")] + + +# --------------------------------------------------------------------------- +# Group 1: Valid baseline (3 tests) +# --------------------------------------------------------------------------- + +class TestValidBaseline: + def test_valid_returns_no_errors(self): + assert _errors(_valid()) == [] + + def test_valid_with_notes(self): + s = _valid_summary(notes="Safety officer reviewed.") + assert _errors(s) == [] + + def test_zero_refusals_valid_except_warning(self): + s = _valid_summary( + total_refused=0, + total_approved=10, + refusal_rate=0.0, + top_refusal_reason="none_refused", + ) + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 2: Rule 1 — rrs_id prefix (4 tests) +# --------------------------------------------------------------------------- + +class TestRrsIdPrefix: + def test_wrong_prefix_rejected(self): + s = _valid_summary(rrs_id="RRF-001") + assert any("rrs_id" in e for e in _errors(s)) + + def test_lowercase_prefix_rejected(self): + s = _valid_summary(rrs_id="rrs-001") + assert any("rrs_id" in e for e in _errors(s)) + + def test_no_prefix_rejected(self): + s = _valid_summary(rrs_id="001") + assert any("rrs_id" in e for e in _errors(s)) + + def test_correct_prefix_accepted(self): + s = _valid_summary(rrs_id="RRS-2024-001") + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 3: Rule 2 — pipeline_version (3 tests) +# --------------------------------------------------------------------------- + +class TestPipelineVersion: + def test_empty_rejected(self): + s = _valid_summary(pipeline_version="") + assert any("pipeline_version" in e for e in _errors(s)) + + def test_whitespace_only_rejected(self): + s = _valid_summary(pipeline_version=" ") + assert any("pipeline_version" in e for e in _errors(s)) + + def test_valid_version(self): + s = _valid_summary(pipeline_version="v3.0.1") + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 4: Rule 3 — period_start ISO format (3 tests) +# --------------------------------------------------------------------------- + +class TestPeriodStart: + def test_invalid_format_rejected(self): + s = _valid_summary(period_start="Jan 2024") + assert any("period_start" in e for e in _errors(s)) + + def test_wrong_separator_rejected(self): + s = _valid_summary(period_start="2024/01/01") + assert any("period_start" in e for e in _errors(s)) + + def test_valid_iso_date(self): + s = _valid_summary(period_start="2024-01-15") + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 5: Rule 4 — period_end ISO format (3 tests) +# --------------------------------------------------------------------------- + +class TestPeriodEnd: + def test_invalid_format_rejected(self): + s = _valid_summary(period_end="Q1 2024") + assert any("period_end" in e for e in _errors(s)) + + def test_empty_rejected(self): + s = _valid_summary(period_end="") + assert any("period_end" in e for e in _errors(s)) + + def test_valid_iso_date(self): + s = _valid_summary(period_end="2024-06-30") + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 6: Rule 5 — total_checkpoints_reviewed >= 1 (3 tests) +# --------------------------------------------------------------------------- + +class TestTotalCheckpointsReviewed: + def test_zero_rejected(self): + s = _valid_summary( + total_checkpoints_reviewed=0, + total_refused=0, + total_approved=0, + refusal_rate=0.0, + ) + assert any("total_checkpoints_reviewed" in e for e in _errors(s)) + + def test_negative_rejected(self): + s = _valid_summary( + total_checkpoints_reviewed=-1, + total_refused=0, + total_approved=0, + refusal_rate=0.0, + ) + assert any("total_checkpoints_reviewed" in e for e in _errors(s)) + + def test_one_accepted(self): + s = _valid_summary( + total_checkpoints_reviewed=1, + total_refused=1, + total_approved=0, + refusal_rate=1.0, + ) + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 7: Rule 6 — total_refused bounds (3 tests) +# --------------------------------------------------------------------------- + +class TestTotalRefused: + def test_negative_rejected(self): + s = _valid_summary( + total_refused=-1, + total_approved=11, + refusal_rate=0.0, + ) + assert any("total_refused" in e for e in _errors(s)) + + def test_exceeds_total_rejected(self): + s = _valid_summary( + total_refused=11, + total_approved=0, + refusal_rate=1.1, + ) + assert any("total_refused" in e for e in _errors(s)) + + def test_exactly_total_accepted(self): + s = _valid_summary( + total_refused=10, + total_approved=0, + refusal_rate=1.0, + ) + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 8: Rule 7 — total_approved bounds (3 tests) +# --------------------------------------------------------------------------- + +class TestTotalApproved: + def test_negative_rejected(self): + s = _valid_summary( + total_refused=10, + total_approved=-1, + refusal_rate=1.0, + ) + assert any("total_approved" in e for e in _errors(s)) + + def test_exceeds_total_rejected(self): + s = _valid_summary( + total_refused=0, + total_approved=11, + refusal_rate=0.0, + ) + assert any("total_approved" in e for e in _errors(s)) + + def test_exactly_total_accepted(self): + s = _valid_summary( + total_refused=0, + total_approved=10, + refusal_rate=0.0, + top_refusal_reason="none_refused", + ) + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 9: Rule 8 — refused + approved == total (3 tests) +# --------------------------------------------------------------------------- + +class TestRefusedPlusApprovedEqualsTotal: + def test_mismatch_rejected(self): + s = _valid_summary( + total_checkpoints_reviewed=10, + total_refused=3, + total_approved=6, + refusal_rate=0.3, + ) + assert any("must equal" in e for e in _errors(s)) + + def test_exact_match_accepted(self): + s = _valid_summary( + total_checkpoints_reviewed=10, + total_refused=4, + total_approved=6, + refusal_rate=0.4, + ) + assert _errors(s) == [] + + def test_all_refused_accepted(self): + s = _valid_summary( + total_checkpoints_reviewed=5, + total_refused=5, + total_approved=0, + refusal_rate=1.0, + ) + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 10: Rule 9 — refusal_rate in [0, 1] (3 tests) +# --------------------------------------------------------------------------- + +class TestRefusalRateRange: + def test_negative_rejected(self): + s = _valid_summary(refusal_rate=-0.1) + assert any("refusal_rate" in e for e in _errors(s)) + + def test_above_one_rejected(self): + s = _valid_summary(refusal_rate=1.1) + assert any("refusal_rate" in e for e in _errors(s)) + + def test_zero_accepted(self): + s = _valid_summary( + total_refused=0, + total_approved=10, + refusal_rate=0.0, + top_refusal_reason="none_refused", + ) + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 11: Rule 10 — refusal_rate consistency (4 tests) +# --------------------------------------------------------------------------- + +class TestRefusalRateConsistency: + def test_inconsistent_rate_rejected(self): + s = _valid_summary( + total_checkpoints_reviewed=10, + total_refused=3, + total_approved=7, + refusal_rate=0.5, + ) + assert any("inconsistent" in e for e in _errors(s)) + + def test_within_tolerance_accepted(self): + s = _valid_summary( + total_checkpoints_reviewed=10, + total_refused=3, + total_approved=7, + refusal_rate=0.305, + ) + assert _errors(s) == [] + + def test_exact_rate_accepted(self): + s = _valid_summary( + total_checkpoints_reviewed=10, + total_refused=3, + total_approved=7, + refusal_rate=0.30, + ) + assert _errors(s) == [] + + def test_just_outside_tolerance_rejected(self): + s = _valid_summary( + total_checkpoints_reviewed=10, + total_refused=3, + total_approved=7, + refusal_rate=0.315, + ) + assert any("inconsistent" in e for e in _errors(s)) + + +# --------------------------------------------------------------------------- +# Group 12: Rule 11 — top_refusal_reason vocabulary (4 tests) +# --------------------------------------------------------------------------- + +class TestTopRefusalReason: + def test_invalid_reason_rejected(self): + s = _valid_summary(top_refusal_reason="made_up_reason") + assert any("top_refusal_reason" in e for e in _errors(s)) + + def test_empty_rejected(self): + s = _valid_summary(top_refusal_reason="") + assert any("top_refusal_reason" in e for e in _errors(s)) + + @pytest.mark.parametrize("reason", sorted(VALID_TOP_REFUSAL_REASONS)) + def test_all_valid_reasons_accepted(self, reason): + s = _valid_summary(top_refusal_reason=reason) + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 13: Rule 12 — gate_status vocabulary (4 tests) +# --------------------------------------------------------------------------- + +class TestGateStatus: + def test_invalid_status_rejected(self): + s = _valid_summary(gate_status="offline") + assert any("gate_status" in e for e in _errors(s)) + + def test_empty_rejected(self): + s = _valid_summary(gate_status="") + assert any("gate_status" in e for e in _errors(s)) + + @pytest.mark.parametrize("status", sorted(VALID_GATE_STATUSES)) + def test_all_valid_statuses_accepted(self, status): + s = _valid_summary(gate_status=status) + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 14: Rule 13 — all_refusals_have_rrf=True (3 tests) +# --------------------------------------------------------------------------- + +class TestAllRefusalsHaveRrf: + def test_false_rejected(self): + s = _valid_summary(all_refusals_have_rrf=False) + assert any("all_refusals_have_rrf" in e for e in _errors(s)) + + def test_true_accepted(self): + s = _valid_summary(all_refusals_have_rrf=True) + assert _errors(s) == [] + + def test_zero_refusals_still_requires_true(self): + s = _valid_summary( + total_refused=0, + total_approved=10, + refusal_rate=0.0, + top_refusal_reason="none_refused", + all_refusals_have_rrf=False, + ) + assert any("all_refusals_have_rrf" in e for e in _errors(s)) + + +# --------------------------------------------------------------------------- +# Group 15: Rule 14 — summary (3 tests) +# --------------------------------------------------------------------------- + +class TestSummary: + def test_empty_rejected(self): + s = _valid_summary(summary="") + assert any("summary" in e for e in _errors(s)) + + def test_too_long_rejected(self): + s = _valid_summary(summary="x" * (SUMMARY_MAX_LENGTH + 1)) + assert any("summary" in e for e in _errors(s)) + + def test_exactly_at_limit_accepted(self): + s = _valid_summary(summary="x" * SUMMARY_MAX_LENGTH) + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 16: Rule 15 — notes length (3 tests) +# --------------------------------------------------------------------------- + +class TestNotesLength: + def test_too_long_rejected(self): + s = _valid_summary(notes="n" * (NOTES_MAX_LENGTH + 1)) + assert any("notes" in e for e in _errors(s)) + + def test_exactly_at_limit_accepted(self): + s = _valid_summary(notes="n" * NOTES_MAX_LENGTH) + assert _errors(s) == [] + + def test_empty_notes_accepted(self): + s = _valid_summary(notes="") + assert _errors(s) == [] + + +# --------------------------------------------------------------------------- +# Group 17: Warnings (6 tests) +# --------------------------------------------------------------------------- + +class TestWarnings: + def test_high_refusal_rate_triggers_warning(self): + s = _valid_summary( + total_refused=9, + total_approved=1, + refusal_rate=0.9, + ) + warns = _warns(s) + assert any("refusal_rate" in w for w in warns) + + def test_zero_refusals_triggers_warning(self): + s = _valid_summary( + total_refused=0, + total_approved=10, + refusal_rate=0.0, + top_refusal_reason="none_refused", + ) + warns = _warns(s) + assert any("total_refused is 0" in w for w in warns) + + def test_empty_notes_triggers_warning(self): + s = _valid_summary(notes="") + warns = _warns(s) + assert any("notes" in w.lower() for w in warns) + + def test_notes_present_suppresses_notes_warning(self): + s = _valid_summary(notes="Reviewed by safety officer.") + warns = _warns(s) + assert not any("notes is empty" in w for w in warns) + + def test_moderate_refusal_rate_no_warning(self): + s = _valid_summary( + total_refused=5, + total_approved=5, + refusal_rate=0.5, + ) + warns = _warns(s) + assert not any("refusal_rate" in w for w in warns) + + def test_at_high_refusal_threshold_boundary(self): + s = _valid_summary( + total_refused=9, + total_approved=1, + refusal_rate=0.9, + ) + warns = _warns(s) + assert any("refusal_rate" in w for w in warns) + + +# --------------------------------------------------------------------------- +# Group 18: validate_dict (4 tests) +# --------------------------------------------------------------------------- + +class TestValidateDict: + def test_valid_dict_returns_no_errors(self): + data = dict( + rrs_id="RRS-20240315-001", + pipeline_version="v2.1.0", + period_start="2024-01-01", + period_end="2024-03-31", + total_checkpoints_reviewed=10, + total_refused=3, + total_approved=7, + refusal_rate=0.30, + top_refusal_reason="effect_within_noise", + gate_status="active", + all_refusals_have_rrf=True, + summary="Gate active; 3/10 refused for valid documented reasons.", + ) + errs = [e for e in validate_dict(data) if not e.startswith("WARNING:")] + assert errs == [] + + def test_missing_required_field_returns_error(self): + data = dict(rrs_id="RRS-001", pipeline_version="v1.0") + result = validate_dict(data) + assert any("Schema construction error" in e for e in result) + + def test_invalid_field_caught_by_dict_validator(self): + data = dict( + rrs_id="WRONG-001", + pipeline_version="v1.0", + period_start="2024-01-01", + period_end="2024-03-31", + total_checkpoints_reviewed=5, + total_refused=2, + total_approved=3, + refusal_rate=0.4, + top_refusal_reason="effect_within_noise", + gate_status="active", + all_refusals_have_rrf=True, + summary="Gate summary.", + ) + result = validate_dict(data) + assert any("rrs_id" in e for e in result) + + def test_extra_notes_in_dict(self): + data = dict( + rrs_id="RRS-20240315-002", + pipeline_version="v2.1.0", + period_start="2024-01-01", + period_end="2024-03-31", + total_checkpoints_reviewed=8, + total_refused=2, + total_approved=6, + refusal_rate=0.25, + top_refusal_reason="insufficient_data", + gate_status="active", + all_refusals_have_rrf=True, + summary="Q1 review: 2 refusals for insufficient data.", + notes="Committee reviewed all refusals.", + ) + errs = [e for e in validate_dict(data) if not e.startswith("WARNING:")] + assert errs == [] + + +# --------------------------------------------------------------------------- +# Group 19: Edge cases (7 tests) +# --------------------------------------------------------------------------- + +class TestEdgeCases: + def test_rrs_prefix_constant(self): + assert RRS_PREFIX == "RRS-" + + def test_refusal_rate_tolerance_constant(self): + assert REFUSAL_RATE_TOLERANCE == 0.01 + + def test_all_valid_gate_statuses(self): + for status in sorted(VALID_GATE_STATUSES): + s = _valid_summary(gate_status=status) + assert _errors(s) == [], f"Status {status} should be valid" + + def test_all_valid_refusal_reasons(self): + for reason in sorted(VALID_TOP_REFUSAL_REASONS): + s = _valid_summary(top_refusal_reason=reason) + assert _errors(s) == [], f"Reason {reason} should be valid" + + def test_suspended_gate_still_valid(self): + s = _valid_summary(gate_status="suspended") + assert _errors(s) == [] + + def test_summary_one_below_limit(self): + s = _valid_summary(summary="s" * (SUMMARY_MAX_LENGTH - 1)) + assert _errors(s) == [] + + def test_notes_one_below_limit(self): + s = _valid_summary(notes="n" * (NOTES_MAX_LENGTH - 1)) + assert _errors(s) == [] diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 8445695b..7ab0a60e 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 5981 +BASELINE = 6044 TOLERANCE = 0.05 # allow 5% fluctuation