diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 9f50867a..311fd777 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -9,6 +9,14 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak ## Changelog +### v0.10.23 — Phase G G8: SyntheticBoundaryAuditRecord schema +- Added `SyntheticBoundaryAuditRecord` (SBR-) schema +- 14 fields: sbr_id, pipeline_version, batch_id, audit_date, evidence_source, total_candidates_checked, total_violations, violation_rate, blocked_upgrades, max_proposed_ladder_level, policy_enforced, enforcement_outcome, summary, notes +- 16 validation rules; 3 warnings; 63 tests +- BASELINE 6044→6107 +- CLI: openamp-foundry synthetic-boundary-audit-record-check +- Closes Phase G G8 — proof-ladder boundary enforcement is now an auditable evidence artifact + ### v0.10.22 — Phase G G2: RecalibrationRejectionSummary schema - Added `RecalibrationRejectionSummary` (RRS-) schema - 13 fields: rrs_id, pipeline_version, period_start, period_end, total_checkpoints_reviewed, total_refused, total_approved, refusal_rate, top_refusal_reason, gate_status, all_refusals_have_rrf, summary, notes diff --git a/docs/evidence/SYNTHETIC_BOUNDARY_AUDIT_RECORD_GUIDE.md b/docs/evidence/SYNTHETIC_BOUNDARY_AUDIT_RECORD_GUIDE.md new file mode 100644 index 00000000..50709408 --- /dev/null +++ b/docs/evidence/SYNTHETIC_BOUNDARY_AUDIT_RECORD_GUIDE.md @@ -0,0 +1,79 @@ +# SyntheticBoundaryAuditRecord (SBR-) Guide + +## Purpose + +`SyntheticBoundaryAuditRecord` makes each enforcement of the synthetic results +policy an auditable artifact. It documents that the boundary between +dry-lab simulation and wet-lab proof is actively enforced — synthetic evidence +cannot raise a candidate's proof-ladder level. + +## Proof-Ladder Boundary + +| Level | Label | Synthetic evidence? | +|-------|-------|---------------------| +| 1 | computational nomination | Yes | +| 2 | virtual-assay support | Yes | +| 3 | in-silico ensemble agreement | Yes | +| **4** | **ex-vivo preliminary** | **No — wet-lab required** | +| 5 | in-vivo preliminary | No — wet-lab required | +| 6 | clinical evidence | No — wet-lab required | + +## Schema Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `sbr_id` | str | yes | Unique identifier, must start with `SBR-` | +| `pipeline_version` | str | yes | Pipeline version during audit | +| `batch_id` | str | yes | Batch identifier audited | +| `audit_date` | str | yes | ISO date (YYYY-MM-DD) of audit | +| `evidence_source` | str | yes | Source type (controlled vocabulary) | +| `total_candidates_checked` | int | yes | Candidates checked (≥1) | +| `total_violations` | int | yes | Candidates with invalid level proposals | +| `violation_rate` | float | yes | Fraction violated (consistency check, tol 0.01) | +| `blocked_upgrades` | int | yes | Violations that were blocked (in [0, total_violations]) | +| `max_proposed_ladder_level` | int | yes | Highest proposed level in batch (1-6) | +| `policy_enforced` | bool | yes | Must be True | +| `enforcement_outcome` | str | yes | Overall outcome (controlled vocabulary) | +| `summary` | str | yes | Narrative summary (≤400 chars) | +| `notes` | str | no | Additional context (≤300 chars) | + +## Controlled Vocabularies + +**evidence_source (4 values):** +- `synthetic`, `simulation`, `computational`, `mixed_synthetic_lab` + +**enforcement_outcome (3 values):** +- `all_passed` — no violations found +- `violations_blocked` — violations found and blocked +- `violations_flagged` — violations found and flagged for review + +## Validation Rules + +1. `sbr_id` starts with `SBR-` +2. `pipeline_version` non-empty +3. `batch_id` non-empty +4. `audit_date` is ISO date (YYYY-MM-DD) +5. `evidence_source` in controlled vocabulary (4 values) +6. `total_candidates_checked` ≥ 1 +7. `total_violations` in [0, total_candidates_checked] +8. `violation_rate` in [0.0, 1.0] +9. `violation_rate` consistent with total_violations/total_candidates_checked (tol 0.01) +10. `blocked_upgrades` in [0, total_violations] +11. `max_proposed_ladder_level` in [1, 6] +12. Synthetic-only sources cannot propose level ≥4 without recording violations +13. `policy_enforced` must be True +14. `enforcement_outcome` in controlled vocabulary (3 values) +15. `summary` non-empty and ≤400 chars +16. `notes` ≤300 chars + +## Warnings + +- `violation_rate > 0.3`: high overclaim attempt rate — review batch scoring +- Violations flagged but not all blocked: verify enforcement_outcome +- `notes` empty: consider documenting enforcement context + +## Integration + +- Triggered after: synthetic score run on candidates +- Violations reference: `SyntheticResultPolicyCheck` (individual policy check) +- Feeds into: evidence audit trail, calibration health, external review packets diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 417be7af..dbbd0207 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -133,7 +133,7 @@ Make learning controlled rather than self-serving. | G5 | Add batch-2 rationale report explaining exploit/explore/diversity roles. | Reviewers understand next batch. | C | | G6 | Add calibration-overfit warning when cohort is too small. | Prevents false learning. | C | | G7 | Add result-quality flag propagation into calibration engine. | Low-quality outcomes cannot drive updates. | C | -| G8 | Add policy that synthetic results cannot raise proof-ladder level. | Anti-overclaim. | B/C | +| G8 | Add policy that synthetic results cannot raise proof-ladder level (complete). | SBR- schema: 14 fields, 16 validation rules, synthetic-only evidence cannot propose level ≥4 without violations recorded, policy_enforced=True enforced, violation rate consistency check (tol 0.01); anti-overclaim boundary is now auditable artifact. | B/C | | G9 | Add calibration decision review checklist. | Human review stronger. | C/D | | G10 | Add recalibration rollback plan. | Safer updates. | C | diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 4ea2b66b..a38aa97d 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -3048,3 +3048,20 @@ def _run_negative_result_dashboard_check(args): for w in result.warnings: print(f"WARNING: {w}") sys.exit(1) + + +def _run_synthetic_boundary_audit_record_check(args): + """CLI handler for synthetic-boundary-audit-record-check.""" + import json, sys + from openamp_foundry.evidence.synthetic_boundary_audit_record import validate_dict + data = json.load(open(args.input)) + issues = validate_dict(data) + errors = [i for i in issues if not i.startswith("WARNING:")] + warnings = [i for i in issues if i.startswith("WARNING:")] + for w in warnings: + print(w) + if errors: + for e in errors: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + print("OK: SyntheticBoundaryAuditRecord is valid.") diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index e31ba201..4087fe31 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -38,6 +38,7 @@ _run_pre_registration_entry_check, _run_recalibration_decision_log_check, _run_recalibration_rejection_summary_check, + _run_synthetic_boundary_audit_record_check, ) from openamp_foundry.cli.commands.gates import _run_gate_check, _run_release_gate_check from openamp_foundry.cli.commands.reports import _run_contribution_check @@ -2538,6 +2539,10 @@ def build_parser() -> argparse.ArgumentParser: p_rrs.add_argument("input", help="Path to JSON file") p_rrs.set_defaults(func=_run_recalibration_rejection_summary_check) + p_sbr = subparsers.add_parser("synthetic-boundary-audit-record-check", help="Validate a SyntheticBoundaryAuditRecord JSON") + p_sbr.add_argument("input", help="Path to JSON file") + p_sbr.set_defaults(func=_run_synthetic_boundary_audit_record_check) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", diff --git a/src/openamp_foundry/evidence/synthetic_boundary_audit_record.py b/src/openamp_foundry/evidence/synthetic_boundary_audit_record.py new file mode 100644 index 00000000..73b22fe9 --- /dev/null +++ b/src/openamp_foundry/evidence/synthetic_boundary_audit_record.py @@ -0,0 +1,216 @@ +"""Synthetic boundary audit record schema — Phase G G8. + +Documents each enforcement of the policy that synthetic/simulation results +cannot raise a candidate's proof-ladder level. + +Proof-ladder levels: + 1 = computational nomination + 2 = virtual-assay support + 3 = in-silico ensemble agreement + 4 = ex-vivo preliminary ← requires wet-lab evidence + 5 = in-vivo preliminary ← requires wet-lab evidence + 6 = clinical evidence ← requires wet-lab evidence + +Synthetic results may maintain or, when combined with prior wet-lab evidence, +annotate a level — but they CANNOT raise it. Level 4+ is unreachable by +simulation alone. + +This schema makes each enforcement event an auditable artifact. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List + +SBR_PREFIX = "SBR-" + +PROOF_LADDER_MIN = 1 +PROOF_LADDER_MAX = 3 # max level reachable via synthetic-only evidence +WET_LAB_REQUIRED_LEVEL = 4 # level ≥ 4 requires wet-lab evidence + +VALID_EVIDENCE_SOURCES = frozenset({ + "synthetic", + "simulation", + "computational", + "mixed_synthetic_lab", +}) + +VALID_ENFORCEMENT_OUTCOMES = frozenset({ + "all_passed", + "violations_blocked", + "violations_flagged", +}) + +VIOLATION_RATE_TOLERANCE = 0.01 +HIGH_VIOLATION_RATE_WARNING = 0.3 +SUMMARY_MAX_LENGTH = 400 +NOTES_MAX_LENGTH = 300 +_ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass +class SyntheticBoundaryAuditRecord: + sbr_id: str + pipeline_version: str + batch_id: str + audit_date: str + evidence_source: str + total_candidates_checked: int + total_violations: int + violation_rate: float + blocked_upgrades: int + max_proposed_ladder_level: int + policy_enforced: bool + enforcement_outcome: str + summary: str + notes: str = "" + + +def validate(record: SyntheticBoundaryAuditRecord) -> List[str]: + """Return list of error strings; empty list means valid.""" + errors: List[str] = [] + + # Rule 1: ID prefix + if not record.sbr_id.startswith(SBR_PREFIX): + errors.append( + f"sbr_id must start with '{SBR_PREFIX}', got: {record.sbr_id!r}" + ) + + # Rule 2: pipeline_version non-empty + if not record.pipeline_version.strip(): + errors.append("pipeline_version must not be empty") + + # Rule 3: batch_id non-empty + if not record.batch_id.strip(): + errors.append("batch_id must not be empty") + + # Rule 4: audit_date ISO format + if not _ISO_DATE_RE.match(record.audit_date): + errors.append( + f"audit_date must be YYYY-MM-DD, got: {record.audit_date!r}" + ) + + # Rule 5: evidence_source vocabulary + if record.evidence_source not in VALID_EVIDENCE_SOURCES: + errors.append( + f"evidence_source must be one of {sorted(VALID_EVIDENCE_SOURCES)}, " + f"got: {record.evidence_source!r}" + ) + + # Rule 6: total_candidates_checked >= 1 + if record.total_candidates_checked < 1: + errors.append( + f"total_candidates_checked must be >= 1, " + f"got: {record.total_candidates_checked}" + ) + + # Rule 7: total_violations in [0, total_candidates_checked] + if not (0 <= record.total_violations <= record.total_candidates_checked): + errors.append( + f"total_violations must be in [0, {record.total_candidates_checked}], " + f"got: {record.total_violations}" + ) + + # Rule 8: violation_rate in [0.0, 1.0] + if not (0.0 <= record.violation_rate <= 1.0): + errors.append( + f"violation_rate must be in [0.0, 1.0], got: {record.violation_rate}" + ) + + # Rule 9: violation_rate consistency (tolerance 0.01) + if record.total_candidates_checked >= 1: + computed = record.total_violations / record.total_candidates_checked + if abs(computed - record.violation_rate) > VIOLATION_RATE_TOLERANCE: + errors.append( + f"violation_rate ({record.violation_rate:.4f}) inconsistent with " + f"total_violations/total_candidates_checked " + f"({record.total_violations}/{record.total_candidates_checked} " + f"= {computed:.4f}); tolerance is {VIOLATION_RATE_TOLERANCE}" + ) + + # Rule 10: blocked_upgrades in [0, total_violations] + if not (0 <= record.blocked_upgrades <= record.total_violations): + errors.append( + f"blocked_upgrades must be in [0, {record.total_violations}], " + f"got: {record.blocked_upgrades}" + ) + + # Rule 11: max_proposed_ladder_level in [1, 6] + if not (1 <= record.max_proposed_ladder_level <= 6): + errors.append( + f"max_proposed_ladder_level must be in [1, 6], " + f"got: {record.max_proposed_ladder_level}" + ) + + # Rule 12: synthetic-only evidence cannot propose level >= 4 + if ( + record.evidence_source in ("synthetic", "simulation", "computational") + and record.max_proposed_ladder_level >= WET_LAB_REQUIRED_LEVEL + and record.total_violations == 0 + ): + errors.append( + f"evidence_source '{record.evidence_source}' cannot support " + f"max_proposed_ladder_level {record.max_proposed_ladder_level} " + f"(level {WET_LAB_REQUIRED_LEVEL}+ requires wet-lab evidence) " + f"without any violations recorded" + ) + + # Rule 13: policy_enforced must be True + if not record.policy_enforced: + errors.append( + "policy_enforced must be True — the synthetic boundary policy " + "must be actively enforced; disable this check requires human review" + ) + + # Rule 14: enforcement_outcome vocabulary + if record.enforcement_outcome not in VALID_ENFORCEMENT_OUTCOMES: + errors.append( + f"enforcement_outcome must be one of " + f"{sorted(VALID_ENFORCEMENT_OUTCOMES)}, " + f"got: {record.enforcement_outcome!r}" + ) + + # Rule 15: summary non-empty and length + if not record.summary.strip(): + errors.append("summary must not be empty") + elif len(record.summary) > SUMMARY_MAX_LENGTH: + errors.append( + f"summary exceeds {SUMMARY_MAX_LENGTH} chars " + f"(got {len(record.summary)})" + ) + + # Rule 16: notes length + if len(record.notes) > NOTES_MAX_LENGTH: + errors.append( + f"notes exceeds {NOTES_MAX_LENGTH} chars (got {len(record.notes)})" + ) + + # Warnings + if record.violation_rate > HIGH_VIOLATION_RATE_WARNING: + errors.append( + f"WARNING: violation_rate ({record.violation_rate:.2f}) > " + f"{HIGH_VIOLATION_RATE_WARNING} — high overclaim attempt rate; " + "review batch selection and scoring methodology" + ) + if record.total_violations > 0 and record.blocked_upgrades < record.total_violations: + errors.append( + f"WARNING: {record.total_violations - record.blocked_upgrades} violations " + "were flagged but not blocked — verify enforcement_outcome is correct" + ) + if not record.notes.strip(): + errors.append( + "WARNING: notes is empty — consider documenting enforcement context" + ) + + return errors + + +def validate_dict(data: dict) -> List[str]: + """Validate a plain dict by constructing SyntheticBoundaryAuditRecord first.""" + try: + record = SyntheticBoundaryAuditRecord(**data) + except TypeError as exc: + return [f"Schema construction error: {exc}"] + return validate(record) diff --git a/tests/evidence/test_synthetic_boundary_audit_record.py b/tests/evidence/test_synthetic_boundary_audit_record.py new file mode 100644 index 00000000..3074fb06 --- /dev/null +++ b/tests/evidence/test_synthetic_boundary_audit_record.py @@ -0,0 +1,607 @@ +"""Tests for SyntheticBoundaryAuditRecord schema — Phase G G8. + +Exactly 63 tests: valid baseline + each validation rule + edge cases + warnings. +""" + +from __future__ import annotations + +import pytest + +from openamp_foundry.evidence.synthetic_boundary_audit_record import ( + NOTES_MAX_LENGTH, + SBR_PREFIX, + SUMMARY_MAX_LENGTH, + VALID_ENFORCEMENT_OUTCOMES, + VALID_EVIDENCE_SOURCES, + VIOLATION_RATE_TOLERANCE, + WET_LAB_REQUIRED_LEVEL, + SyntheticBoundaryAuditRecord, + validate, + validate_dict, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _valid_record(**overrides) -> SyntheticBoundaryAuditRecord: + defaults = dict( + sbr_id="SBR-20240315-001", + pipeline_version="v2.1.0", + batch_id="BATCH-003", + audit_date="2024-03-15", + evidence_source="synthetic", + total_candidates_checked=20, + total_violations=2, + violation_rate=0.10, + blocked_upgrades=2, + max_proposed_ladder_level=3, + policy_enforced=True, + enforcement_outcome="violations_blocked", + summary="2 of 20 candidates attempted level 4 upgrade; blocked by policy.", + notes="", + ) + defaults.update(overrides) + return SyntheticBoundaryAuditRecord(**defaults) + + +def _valid() -> SyntheticBoundaryAuditRecord: + return _valid_record() + + +def _errors(r): + return [e for e in validate(r) if not e.startswith("WARNING:")] + + +def _warns(r): + return [e for e in validate(r) if e.startswith("WARNING:")] + + +# --------------------------------------------------------------------------- +# Group 1: Valid baseline (3 tests) +# --------------------------------------------------------------------------- + +class TestValidBaseline: + def test_valid_returns_no_errors(self): + assert _errors(_valid()) == [] + + def test_valid_no_violations(self): + r = _valid_record( + total_violations=0, + violation_rate=0.0, + blocked_upgrades=0, + enforcement_outcome="all_passed", + ) + assert _errors(r) == [] + + def test_valid_with_notes(self): + r = _valid_record(notes="Reviewed by safety officer.") + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 2: Rule 1 — sbr_id prefix (4 tests) +# --------------------------------------------------------------------------- + +class TestSbrIdPrefix: + def test_wrong_prefix_rejected(self): + r = _valid_record(sbr_id="CIR-001") + assert any("sbr_id" in e for e in _errors(r)) + + def test_lowercase_prefix_rejected(self): + r = _valid_record(sbr_id="sbr-001") + assert any("sbr_id" in e for e in _errors(r)) + + def test_no_prefix_rejected(self): + r = _valid_record(sbr_id="001") + assert any("sbr_id" in e for e in _errors(r)) + + def test_correct_prefix_accepted(self): + r = _valid_record(sbr_id="SBR-2024-001") + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 3: Rule 2 — pipeline_version (3 tests) +# --------------------------------------------------------------------------- + +class TestPipelineVersion: + def test_empty_rejected(self): + r = _valid_record(pipeline_version="") + assert any("pipeline_version" in e for e in _errors(r)) + + def test_whitespace_only_rejected(self): + r = _valid_record(pipeline_version=" ") + assert any("pipeline_version" in e for e in _errors(r)) + + def test_valid_version(self): + r = _valid_record(pipeline_version="v3.0.1") + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 4: Rule 3 — batch_id (3 tests) +# --------------------------------------------------------------------------- + +class TestBatchId: + def test_empty_rejected(self): + r = _valid_record(batch_id="") + assert any("batch_id" in e for e in _errors(r)) + + def test_whitespace_only_rejected(self): + r = _valid_record(batch_id=" ") + assert any("batch_id" in e for e in _errors(r)) + + def test_valid_batch_id(self): + r = _valid_record(batch_id="BSP-20240101-001") + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 5: Rule 4 — audit_date ISO format (3 tests) +# --------------------------------------------------------------------------- + +class TestAuditDate: + def test_invalid_format_rejected(self): + r = _valid_record(audit_date="March 15, 2024") + assert any("audit_date" in e for e in _errors(r)) + + def test_wrong_separator_rejected(self): + r = _valid_record(audit_date="2024/03/15") + assert any("audit_date" in e for e in _errors(r)) + + def test_valid_iso_date(self): + r = _valid_record(audit_date="2025-01-01") + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 6: Rule 5 — evidence_source vocabulary (4 tests) +# --------------------------------------------------------------------------- + +class TestEvidenceSource: + def test_invalid_source_rejected(self): + r = _valid_record(evidence_source="wet_lab") + assert any("evidence_source" in e for e in _errors(r)) + + def test_empty_rejected(self): + r = _valid_record(evidence_source="") + assert any("evidence_source" in e for e in _errors(r)) + + @pytest.mark.parametrize("src", sorted(VALID_EVIDENCE_SOURCES)) + def test_all_valid_sources_accepted(self, src): + extra = {} + if src == "mixed_synthetic_lab": + extra["max_proposed_ladder_level"] = 3 + r = _valid_record(evidence_source=src, **extra) + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 7: Rule 6 — total_candidates_checked >= 1 (3 tests) +# --------------------------------------------------------------------------- + +class TestTotalCandidatesChecked: + def test_zero_rejected(self): + r = _valid_record( + total_candidates_checked=0, + total_violations=0, + violation_rate=0.0, + blocked_upgrades=0, + ) + assert any("total_candidates_checked" in e for e in _errors(r)) + + def test_negative_rejected(self): + r = _valid_record( + total_candidates_checked=-5, + total_violations=0, + violation_rate=0.0, + blocked_upgrades=0, + ) + assert any("total_candidates_checked" in e for e in _errors(r)) + + def test_one_accepted(self): + r = _valid_record( + total_candidates_checked=1, + total_violations=0, + violation_rate=0.0, + blocked_upgrades=0, + enforcement_outcome="all_passed", + ) + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 8: Rule 7 — total_violations bounds (3 tests) +# --------------------------------------------------------------------------- + +class TestTotalViolations: + def test_negative_rejected(self): + r = _valid_record( + total_violations=-1, + violation_rate=0.0, + blocked_upgrades=0, + ) + assert any("total_violations" in e for e in _errors(r)) + + def test_exceeds_total_rejected(self): + r = _valid_record( + total_violations=21, + violation_rate=1.05, + blocked_upgrades=21, + ) + assert any("total_violations" in e for e in _errors(r)) + + def test_exactly_total_accepted(self): + r = _valid_record( + total_violations=20, + violation_rate=1.0, + blocked_upgrades=20, + ) + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 9: Rule 8 — violation_rate range (3 tests) +# --------------------------------------------------------------------------- + +class TestViolationRateRange: + def test_negative_rejected(self): + r = _valid_record(violation_rate=-0.1) + assert any("violation_rate" in e for e in _errors(r)) + + def test_above_one_rejected(self): + r = _valid_record(violation_rate=1.1) + assert any("violation_rate" in e for e in _errors(r)) + + def test_zero_accepted(self): + r = _valid_record( + total_violations=0, + violation_rate=0.0, + blocked_upgrades=0, + enforcement_outcome="all_passed", + ) + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 10: Rule 9 — violation_rate consistency (4 tests) +# --------------------------------------------------------------------------- + +class TestViolationRateConsistency: + def test_inconsistent_rate_rejected(self): + r = _valid_record( + total_candidates_checked=20, + total_violations=2, + violation_rate=0.5, + blocked_upgrades=2, + ) + assert any("inconsistent" in e for e in _errors(r)) + + def test_exact_rate_accepted(self): + r = _valid_record( + total_candidates_checked=20, + total_violations=2, + violation_rate=0.10, + blocked_upgrades=2, + ) + assert _errors(r) == [] + + def test_within_tolerance_accepted(self): + r = _valid_record( + total_candidates_checked=20, + total_violations=2, + violation_rate=0.105, + blocked_upgrades=2, + ) + assert _errors(r) == [] + + def test_just_outside_tolerance_rejected(self): + r = _valid_record( + total_candidates_checked=20, + total_violations=2, + violation_rate=0.115, + blocked_upgrades=2, + ) + assert any("inconsistent" in e for e in _errors(r)) + + +# --------------------------------------------------------------------------- +# Group 11: Rule 10 — blocked_upgrades bounds (3 tests) +# --------------------------------------------------------------------------- + +class TestBlockedUpgrades: + def test_negative_rejected(self): + r = _valid_record(blocked_upgrades=-1) + assert any("blocked_upgrades" in e for e in _errors(r)) + + def test_exceeds_violations_rejected(self): + r = _valid_record( + total_violations=2, + blocked_upgrades=3, + ) + assert any("blocked_upgrades" in e for e in _errors(r)) + + def test_zero_blocked_when_no_violations_accepted(self): + r = _valid_record( + total_violations=0, + violation_rate=0.0, + blocked_upgrades=0, + enforcement_outcome="all_passed", + ) + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 12: Rule 11 — max_proposed_ladder_level range (3 tests) +# --------------------------------------------------------------------------- + +class TestMaxProposedLadderLevel: + def test_zero_rejected(self): + r = _valid_record(max_proposed_ladder_level=0) + assert any("max_proposed_ladder_level" in e for e in _errors(r)) + + def test_seven_rejected(self): + r = _valid_record(max_proposed_ladder_level=7) + assert any("max_proposed_ladder_level" in e for e in _errors(r)) + + def test_level_three_accepted(self): + r = _valid_record(max_proposed_ladder_level=3) + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 13: Rule 12 — synthetic cannot propose level 4+ without violations (3 tests) +# --------------------------------------------------------------------------- + +class TestSyntheticLevelBoundary: + def test_synthetic_proposing_level4_without_violations_rejected(self): + r = _valid_record( + evidence_source="synthetic", + max_proposed_ladder_level=4, + total_violations=0, + violation_rate=0.0, + blocked_upgrades=0, + enforcement_outcome="all_passed", + ) + assert any(str(WET_LAB_REQUIRED_LEVEL) in e for e in _errors(r)) + + def test_synthetic_proposing_level4_with_violations_accepted(self): + r = _valid_record( + evidence_source="synthetic", + max_proposed_ladder_level=4, + total_violations=1, + violation_rate=0.05, + blocked_upgrades=1, + ) + assert _errors(r) == [] + + def test_mixed_source_proposing_level4_accepted(self): + r = _valid_record( + evidence_source="mixed_synthetic_lab", + max_proposed_ladder_level=4, + total_violations=0, + violation_rate=0.0, + blocked_upgrades=0, + enforcement_outcome="all_passed", + ) + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 14: Rule 13 — policy_enforced (3 tests) +# --------------------------------------------------------------------------- + +class TestPolicyEnforced: + def test_false_rejected(self): + r = _valid_record(policy_enforced=False) + assert any("policy_enforced" in e for e in _errors(r)) + + def test_true_accepted(self): + r = _valid_record(policy_enforced=True) + assert _errors(r) == [] + + def test_enforcement_constant_check(self): + assert WET_LAB_REQUIRED_LEVEL == 4 + + +# --------------------------------------------------------------------------- +# Group 15: Rule 14 — enforcement_outcome vocabulary (4 tests) +# --------------------------------------------------------------------------- + +class TestEnforcementOutcome: + def test_invalid_outcome_rejected(self): + r = _valid_record(enforcement_outcome="partial") + assert any("enforcement_outcome" in e for e in _errors(r)) + + def test_empty_rejected(self): + r = _valid_record(enforcement_outcome="") + assert any("enforcement_outcome" in e for e in _errors(r)) + + @pytest.mark.parametrize("outcome", sorted(VALID_ENFORCEMENT_OUTCOMES)) + def test_all_valid_outcomes_accepted(self, outcome): + r = _valid_record(enforcement_outcome=outcome) + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 16: Rule 15+16 — summary and notes (4 tests) +# --------------------------------------------------------------------------- + +class TestSummaryAndNotes: + def test_empty_summary_rejected(self): + r = _valid_record(summary="") + assert any("summary" in e for e in _errors(r)) + + def test_summary_too_long_rejected(self): + r = _valid_record(summary="x" * (SUMMARY_MAX_LENGTH + 1)) + assert any("summary" in e for e in _errors(r)) + + def test_notes_too_long_rejected(self): + r = _valid_record(notes="n" * (NOTES_MAX_LENGTH + 1)) + assert any("notes" in e for e in _errors(r)) + + def test_summary_at_limit_accepted(self): + r = _valid_record(summary="x" * SUMMARY_MAX_LENGTH) + assert _errors(r) == [] + + +# --------------------------------------------------------------------------- +# Group 17: Warnings (6 tests) +# --------------------------------------------------------------------------- + +class TestWarnings: + def test_high_violation_rate_triggers_warning(self): + r = _valid_record( + total_violations=8, + violation_rate=0.4, + blocked_upgrades=8, + ) + warns = _warns(r) + assert any("violation_rate" in w for w in warns) + + def test_partial_block_triggers_warning(self): + r = _valid_record( + total_violations=2, + violation_rate=0.10, + blocked_upgrades=1, + enforcement_outcome="violations_flagged", + ) + warns = _warns(r) + assert any("flagged but not blocked" in w for w in warns) + + def test_empty_notes_triggers_warning(self): + r = _valid_record(notes="") + warns = _warns(r) + assert any("notes" in w.lower() for w in warns) + + def test_notes_present_suppresses_notes_warning(self): + r = _valid_record(notes="Reviewed and signed off.") + warns = _warns(r) + assert not any("notes is empty" in w for w in warns) + + def test_low_violation_rate_no_high_rate_warning(self): + r = _valid_record( + total_violations=1, + violation_rate=0.05, + blocked_upgrades=1, + ) + warns = _warns(r) + assert not any("high overclaim" in w for w in warns) + + def test_all_blocked_no_partial_warning(self): + r = _valid_record( + total_violations=2, + violation_rate=0.10, + blocked_upgrades=2, + enforcement_outcome="violations_blocked", + ) + warns = _warns(r) + assert not any("flagged but not blocked" in w for w in warns) + + +# --------------------------------------------------------------------------- +# Group 18: validate_dict (4 tests) +# --------------------------------------------------------------------------- + +class TestValidateDict: + def test_valid_dict_returns_no_errors(self): + data = dict( + sbr_id="SBR-20240315-001", + pipeline_version="v2.1.0", + batch_id="BATCH-003", + audit_date="2024-03-15", + evidence_source="synthetic", + total_candidates_checked=20, + total_violations=2, + violation_rate=0.10, + blocked_upgrades=2, + max_proposed_ladder_level=3, + policy_enforced=True, + enforcement_outcome="violations_blocked", + summary="2 of 20 candidates attempted invalid level upgrade; blocked.", + ) + errs = [e for e in validate_dict(data) if not e.startswith("WARNING:")] + assert errs == [] + + def test_missing_required_field_returns_error(self): + data = dict(sbr_id="SBR-001", pipeline_version="v1.0") + result = validate_dict(data) + assert any("Schema construction error" in e for e in result) + + def test_invalid_field_caught_by_dict_validator(self): + data = dict( + sbr_id="WRONG-001", + pipeline_version="v1.0", + batch_id="BATCH-001", + audit_date="2024-01-01", + evidence_source="synthetic", + total_candidates_checked=10, + total_violations=0, + violation_rate=0.0, + blocked_upgrades=0, + max_proposed_ladder_level=3, + policy_enforced=True, + enforcement_outcome="all_passed", + summary="No violations.", + ) + result = validate_dict(data) + assert any("sbr_id" in e for e in result) + + def test_extra_notes_in_dict(self): + data = dict( + sbr_id="SBR-20240315-002", + pipeline_version="v2.1.0", + batch_id="BATCH-004", + audit_date="2024-03-15", + evidence_source="simulation", + total_candidates_checked=15, + total_violations=0, + violation_rate=0.0, + blocked_upgrades=0, + max_proposed_ladder_level=2, + policy_enforced=True, + enforcement_outcome="all_passed", + summary="All 15 candidates within level 1-3 as expected.", + notes="Clean batch, no interventions needed.", + ) + errs = [e for e in validate_dict(data) if not e.startswith("WARNING:")] + assert errs == [] + + +# --------------------------------------------------------------------------- +# Group 19: Edge cases (7 tests) +# --------------------------------------------------------------------------- + +class TestEdgeCases: + def test_sbr_prefix_constant(self): + assert SBR_PREFIX == "SBR-" + + def test_violation_rate_tolerance_constant(self): + assert VIOLATION_RATE_TOLERANCE == 0.01 + + def test_all_valid_evidence_sources(self): + for src in sorted(VALID_EVIDENCE_SOURCES): + extra = {} + if src == "mixed_synthetic_lab": + extra["max_proposed_ladder_level"] = 3 + r = _valid_record(evidence_source=src, **extra) + assert _errors(r) == [], f"Source {src} should be valid" + + def test_all_valid_enforcement_outcomes(self): + for outcome in sorted(VALID_ENFORCEMENT_OUTCOMES): + r = _valid_record(enforcement_outcome=outcome) + assert _errors(r) == [], f"Outcome {outcome} should be valid" + + def test_simulation_source_valid(self): + r = _valid_record(evidence_source="simulation") + assert _errors(r) == [] + + def test_summary_one_below_limit(self): + r = _valid_record(summary="s" * (SUMMARY_MAX_LENGTH - 1)) + assert _errors(r) == [] + + def test_notes_one_below_limit(self): + r = _valid_record(notes="n" * (NOTES_MAX_LENGTH - 1)) + assert _errors(r) == [] diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 7ab0a60e..b576f79b 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 6044 +BASELINE = 6107 TOLERANCE = 0.05 # allow 5% fluctuation