diff --git a/Makefile b/Makefile index 2493ce2b..d329d7a8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -880,3 +880,8 @@ advisory-review-check: .venv/bin/openamp-foundry advisory-review-check \ --review-json '{"review_id":"ADV-2026-001","review_type":"candidate_review","artifact_id":"OAMP-v0.7.6-candidates","reviewer_handle":"external-advisor-1","assigned_date":"2026-07-09","deadline_date":"2026-07-23","status":"pending"}' @echo "Advisory review check complete." + +annual-review-check: + .venv/bin/openamp-foundry annual-review-check \ + --entry-json '{"review_id":"ANN-2026-001","year":"2026","section":"safety_policy","reviewer":"maintainer-1","finding_count":0,"action_items_count":0,"status":"completed","notes":"All safety checks passed.","completion_date":"2026-07-09"}' + @echo "Annual review check complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 39415c45..7fb5a1c0 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,9 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase J J9 — external advisory review process — v0.7.7) +> **Last updated:** 2026-07-09 (Phase J J10 — annual safety and benchmark review checklist — v0.7.8) +> **New in v0.7.8:** Annual safety and benchmark review checklist (J10 — Loop 118) — docs/governance/ANNUAL_REVIEW_CHECKLIST.md with 5-section structured checklist (safety_policy: 6 checks including dual-use safeguards, dry_lab_only enforcement, toxicity filters; benchmark_thresholds: 6 checks including threshold loosening guard, easy-baseline requirement, deprecation check; calibration_status: 4 checks including recalibration gate and rollback plan; governance_decisions: 4 checks including COI disclosures and rotation plan; data_governance: 3 checks including proprietary data flag). src/openamp_foundry/governance/annual_review.py with AnnualReviewEntry dataclass (10 fields: review_id, year, section, reviewer, finding_count, action_items_count, status, notes, completion_date, dry_lab_only), AnnualReviewResult dataclass (6 fields: review_id, year, section, passed, errors, warnings, dry_lab_only=True), VALID_REVIEW_SECTIONS (5: benchmark_thresholds, calibration_status, data_governance, governance_decisions, safety_policy), VALID_ENTRY_STATUSES (5: completed, deferred, in_progress, not_applicable, pending), validate_annual_review_entry() (9 checks: ANN- prefix, 4-digit year, valid section, non-empty reviewer, non-negative finding_count, non-negative action_items_count, valid status, completed requires YYYY-MM-DD completion_date, dry_lab_only must be True; completed+no-notes warns, deferred warns, findings+no-action-items warns), validate_annual_review_dict() (7 required fields guard). CLI (openamp-foundry annual-review-check) with --entry-json, --format text|json. make annual-review-check target. Long-term trust: annual review entries are now machine-validated. +> > **New in v0.7.7:** External advisory review process (J9 — Loop 117) — docs/governance/EXTERNAL_ADVISORY_REVIEW_PROCESS.md with reviewer eligibility criteria, review scope table (5 review types with minimum reviewer counts), 5-step review process (prepare packet, assign+disclose, receive+log, respond to findings, close), finding severity classification (critical/major/minor/informational), limitations. src/openamp_foundry/governance/advisory_review.py with AdvisoryReview dataclass (11 fields: review_id, review_type, artifact_id, reviewer_handle, assigned_date, deadline_date, status, finding_severity, finding_summary, resolved, dry_lab_only), AdvisoryReviewResult dataclass (5 fields: review_id, review_type, passed, errors, warnings, dry_lab_only=True), VALID_REVIEW_TYPES (5: benchmark_audit, candidate_review, evidence_review, governance_review, safety_policy_review), VALID_REVIEW_STATUSES (5: assigned, completed, deferred, in_progress, pending), VALID_FINDING_SEVERITIES (4: critical, informational, major, minor), MINIMUM_REVIEWER_COUNTS (5 entries: candidate_review and safety_policy_review require 2, others 1), validate_advisory_review() (9 checks: ADV- prefix, valid review_type, non-empty artifact_id, non-empty reviewer_handle, YYYY-MM-DD dates, valid status, valid finding_severity, dry_lab_only must be True; critical+unresolved warns, completed+no-summary warns, deferred warns), validate_advisory_review_dict() (7 required fields guard). CLI (openamp-foundry advisory-review-check) with --review-json, --format text|json. make advisory-review-check target. 29 tests. Credibility: external advisory reviews now have a validated structure and documented process. > > **New in v0.7.6:** Roadmap-to-issue sync checklist (J8 — Loop 116) — docs/governance/ROADMAP_ISSUE_SYNC_CHECKLIST.md with 5-section sync checklist (roadmap items → issues, issues → roadmap, completed items, priority alignment, version consistency). src/openamp_foundry/governance/roadmap_sync.py with RoadmapSyncEntry dataclass (10 fields: item_id, phase, description, priority, sync_status, issue_number, pr_number, completed, completion_date, dry_lab_only), RoadmapSyncResult dataclass (5 fields: item_id, phase, passed, errors, warnings, dry_lab_only=True), VALID_SYNC_STATUSES (5: synced, missing_issue, orphaned_issue, stale, completed), VALID_PRIORITY_LEVELS (4: A, B, C, D), VALID_PHASES (7: E, F, G, H, I, J, K), validate_roadmap_sync_entry() (8 checks: non-empty item_id, valid phase, non-empty description, valid priority, valid sync_status, dry_lab_only must be True, completed must have completion_date, completion_date must be YYYY-MM-DD; priority A + missing_issue warns, stale warns, orphaned_issue warns, no issue_number warns), validate_roadmap_sync_dict() (5 required fields guard). CLI (openamp-foundry roadmap-sync-check) with --entry-json, --format text|json. make roadmap-sync-check target. 24 tests. Keeps strategy actionable: roadmap sync entries are now machine-validated. diff --git a/docs/governance/ANNUAL_REVIEW_CHECKLIST.md b/docs/governance/ANNUAL_REVIEW_CHECKLIST.md new file mode 100644 index 00000000..a039a885 --- /dev/null +++ b/docs/governance/ANNUAL_REVIEW_CHECKLIST.md @@ -0,0 +1,56 @@ +# Annual Safety and Benchmark Review Checklist + +> **Scope:** Dry-lab only. No wet-lab or in-vivo claims. +> **Cadence:** Annually, or after any major pipeline version change. +> **Purpose:** Ensure safety policy, benchmark thresholds, calibration, governance decisions, +> and data governance remain current and honest. + +--- + +## 1. Safety Policy Review + +- [ ] Re-read `SAFETY.md` and confirm all dual-use safeguards remain in effect. +- [ ] Confirm `dry_lab_only: True` is enforced on all result and report dataclasses. +- [ ] Confirm that `evidence_level > 4` with `dry_lab_only=True` still raises an error. +- [ ] Confirm public releases with `safety_review_status: pending` are still blocked. +- [ ] Confirm the toxicity and hemolysis filter thresholds have not been weakened. +- [ ] Document any safety policy changes with a new `GOV-XXX` decision log entry. + +## 2. Benchmark Threshold Review + +- [ ] Re-read `docs/evidence/BENCHMARK_GOVERNANCE.md` for threshold rationale. +- [ ] Confirm benchmark thresholds have not been silently loosened. +- [ ] Check whether any benchmark has been deprecated without replacement. +- [ ] Confirm all benchmarks include an easy-baseline comparison. +- [ ] Confirm selectivity benchmarks are still present and passing. +- [ ] Document any threshold changes with a `GOV-XXX` decision log entry and rationale. + +## 3. Calibration Status Review + +- [ ] Run `make bench-calibration` and confirm no calibration regression. +- [ ] Confirm the calibration decision checklist has been reviewed. +- [ ] Confirm the recalibration gate is still active. +- [ ] Confirm calibration rollback plan is documented and up to date. + +## 4. Governance Decisions Review + +- [ ] Run `make decision-log` and confirm all governance decisions are `active` or `superseded` with rationale. +- [ ] Confirm no governance decision is in `under_review` status for more than 30 days without action. +- [ ] Confirm COI disclosures are filed for all maintainers and external advisors. +- [ ] Confirm maintainer rotation plan is up to date (`make rotation-plan-check`). + +## 5. Data Governance Review + +- [ ] Confirm `DATA_LICENSE_NOTICE.md` is current. +- [ ] Confirm no proprietary data has entered the pipeline without a `contact_required` license flag. +- [ ] Confirm all external data sources are documented in the data governance log. + +## 6. Action Items + +After completing each section, record findings using `make annual-review-check`. +Any `finding_count > 0` must have a corresponding `action_items_count > 0`. +Critical findings halt the next candidate release until resolved. + +--- + +*This checklist is machine-validated. See `src/openamp_foundry/governance/annual_review.py`.* diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 657b02ad..2eaacdd7 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,33 @@ # Roadmap +## v0.7.8 — Loop 118: Phase J J10 — Annual Safety and Benchmark Review Checklist ✓ (2026-07-09) + +`docs/governance/ANNUAL_REVIEW_CHECKLIST.md` with 5-section structured annual +review checklist (safety_policy: 6 checks covering dual-use safeguards, +dry_lab_only enforcement, toxicity/hemolysis filter thresholds, evidence_level +guard; benchmark_thresholds: 6 checks covering threshold loosening guard, +easy-baseline requirement, selectivity benchmarks, deprecation check; +calibration_status: 4 checks covering recalibration gate, decision checklist, +rollback plan; governance_decisions: 4 checks covering active decisions, +COI disclosures, maintainer rotation; data_governance: 3 checks covering +proprietary data license flags, external source documentation). + +`src/openamp_foundry/governance/annual_review.py` with `AnnualReviewEntry` +dataclass (10 fields: review_id, year, section, reviewer, finding_count, +action_items_count, status, notes, completion_date, dry_lab_only), +`AnnualReviewResult` dataclass (6 fields, dry_lab_only=True), +`VALID_REVIEW_SECTIONS` (5: benchmark_thresholds, calibration_status, +data_governance, governance_decisions, safety_policy), +`VALID_ENTRY_STATUSES` (5: completed, deferred, in_progress, not_applicable, +pending), `validate_annual_review_entry()` (9 checks: ANN- prefix, 4-digit year, +valid section, non-empty reviewer, non-negative finding_count, +non-negative action_items_count, valid status, completed requires YYYY-MM-DD +completion_date, dry_lab_only must be True; completed+no-notes warns, deferred +warns, findings+no-action-items warns), `validate_annual_review_dict()` (7 +required fields guard). CLI (`openamp-foundry annual-review-check`) with +`--entry-json`, `--format text|json`. `make annual-review-check` target. +Long-term trust: annual review entries are now machine-validated. + ## v0.7.7 — Loop 117: Phase J J9 — External Advisory Review Process ✓ (2026-07-09) `docs/governance/EXTERNAL_ADVISORY_REVIEW_PROCESS.md` with reviewer eligibility diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 87ac83c2..d5dbdb9d 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -2783,3 +2783,37 @@ def _run_roadmap_sync_check(args: argparse.Namespace) -> int: print(" Roadmap sync entry validated. Strategy is actionable.") return 0 if result.passed else 3 + + +def _run_annual_review_check(args: argparse.Namespace) -> int: + """Validate an annual review checklist entry.""" + import json as _json + from openamp_foundry.governance.annual_review import validate_annual_review_dict + + try: + d = _json.loads(args.entry_json) + except _json.JSONDecodeError as exc: + print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"})) + return 2 + + if not isinstance(d, dict): + print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"})) + return 2 + + result = validate_annual_review_dict(d) + output_format = getattr(args, "format", "text") + + if output_format == "json": + import dataclasses + print(_json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + print(f"Annual review {result.review_id} ({result.section} / {result.year}): {status}") + for e in result.errors: + print(f" ERROR: {e}") + for w in result.warnings: + print(f" WARN: {w}") + if result.passed: + print(" Annual review entry validated. Long-term trust maintained.") + + return 0 if result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index f15baa98..d490a0e5 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -15,7 +15,7 @@ from openamp_foundry.cli.commands.reports import _run_security_report_check from openamp_foundry.cli.commands.reports import _run_citation_check from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check -from openamp_foundry.cli.commands.reports import _run_advisory_review_check +from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check import argparse import json @@ -1999,6 +1999,24 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Annual review check (Phase J J10) ─────────────────────────── + anrc = sub.add_parser( + "annual-review-check", + help=( + "Validate an annual safety and benchmark review checklist entry. " + "Dry-lab only." + ), + ) + anrc.add_argument( + "--entry-json", type=str, required=True, + help="JSON string of an AnnualReviewEntry dict (required).", + ) + anrc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Simulation-result confidence interval report (H7) ──────────── sicr = sub.add_parser( "simulation-ci-report", @@ -2264,6 +2282,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "advisory-review-check": return _run_advisory_review_check(args) + if args.command == "annual-review-check": + return _run_annual_review_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/governance/annual_review.py b/src/openamp_foundry/governance/annual_review.py new file mode 100644 index 00000000..5d50559e --- /dev/null +++ b/src/openamp_foundry/governance/annual_review.py @@ -0,0 +1,171 @@ +"""Annual safety and benchmark review checklist validation for OpenAMP Foundry.""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List + +VALID_REVIEW_SECTIONS: set[str] = { + "benchmark_thresholds", + "calibration_status", + "data_governance", + "governance_decisions", + "safety_policy", +} +VALID_ENTRY_STATUSES: set[str] = { + "completed", + "deferred", + "in_progress", + "not_applicable", + "pending", +} + + +@dataclass +class AnnualReviewEntry: + """A single annual review checklist entry for one section.""" + + review_id: str + year: str + section: str + reviewer: str + finding_count: int + action_items_count: int + status: str + notes: str = "" + completion_date: str = "" + dry_lab_only: bool = True + + +@dataclass +class AnnualReviewResult: + """Result of validating an AnnualReviewEntry.""" + + review_id: str + year: str + section: str + passed: bool + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + dry_lab_only: bool = True + + +def validate_annual_review_entry(entry: AnnualReviewEntry) -> AnnualReviewResult: + """Validate an AnnualReviewEntry against policy rules.""" + errors: list[str] = [] + warnings: list[str] = [] + + if not entry.review_id or not str(entry.review_id).strip(): + errors.append("review_id must be non-empty") + elif not entry.review_id.startswith("ANN-"): + errors.append( + f"review_id must start with 'ANN-', got: {entry.review_id!r}" + ) + + if not re.match(r"^\d{4}$", str(entry.year)): + errors.append( + f"year must be a 4-digit string, got: {entry.year!r}" + ) + + if entry.section not in VALID_REVIEW_SECTIONS: + errors.append( + f"section must be one of {sorted(VALID_REVIEW_SECTIONS)}, " + f"got: {entry.section!r}" + ) + + if not entry.reviewer or not str(entry.reviewer).strip(): + errors.append("reviewer must be non-empty") + + if not isinstance(entry.finding_count, int) or entry.finding_count < 0: + errors.append( + f"finding_count must be a non-negative integer, got: {entry.finding_count!r}" + ) + + if not isinstance(entry.action_items_count, int) or entry.action_items_count < 0: + errors.append( + f"action_items_count must be a non-negative integer, " + f"got: {entry.action_items_count!r}" + ) + + if entry.status not in VALID_ENTRY_STATUSES: + errors.append( + f"status must be one of {sorted(VALID_ENTRY_STATUSES)}, " + f"got: {entry.status!r}" + ) + + if entry.status == "completed" and not re.match(r"^\d{4}-\d{2}-\d{2}$", entry.completion_date): + errors.append( + f"completion_date must be YYYY-MM-DD when status is 'completed', " + f"got: {entry.completion_date!r}" + ) + + if not entry.dry_lab_only: + errors.append("dry_lab_only must be True for annual review entries") + + # Warnings + if entry.status == "completed" and not entry.notes: + warnings.append( + "status is 'completed' but notes is empty: document the review outcome" + ) + + if entry.status == "deferred": + warnings.append( + "review is 'deferred': document the reason and reschedule before next release" + ) + + if ( + isinstance(entry.finding_count, int) + and entry.finding_count > 0 + and isinstance(entry.action_items_count, int) + and entry.action_items_count == 0 + ): + warnings.append( + f"finding_count is {entry.finding_count} but action_items_count is 0: " + "findings must have corresponding action items" + ) + + return AnnualReviewResult( + review_id=entry.review_id, + year=entry.year, + section=entry.section, + passed=len(errors) == 0, + errors=errors, + warnings=warnings, + dry_lab_only=True, + ) + + +def validate_annual_review_dict(d: dict) -> AnnualReviewResult: + """Validate a dict representation of an AnnualReviewEntry.""" + required = [ + "action_items_count", + "finding_count", + "review_id", + "reviewer", + "section", + "status", + "year", + ] + missing = [f for f in required if f not in d] + if missing: + return AnnualReviewResult( + review_id=d.get("review_id", ""), + year=d.get("year", ""), + section=d.get("section", ""), + passed=False, + errors=[f"Missing required fields: {missing}"], + dry_lab_only=True, + ) + entry = AnnualReviewEntry( + review_id=d["review_id"], + year=d["year"], + section=d["section"], + reviewer=d["reviewer"], + finding_count=d["finding_count"], + action_items_count=d["action_items_count"], + status=d["status"], + notes=d.get("notes", ""), + completion_date=d.get("completion_date", ""), + dry_lab_only=d.get("dry_lab_only", True), + ) + return validate_annual_review_entry(entry) diff --git a/tests/governance/test_annual_review.py b/tests/governance/test_annual_review.py new file mode 100644 index 00000000..bf980ae7 --- /dev/null +++ b/tests/governance/test_annual_review.py @@ -0,0 +1,211 @@ +"""Tests for annual safety and benchmark review checklist validation.""" +import pytest +from openamp_foundry.governance.annual_review import ( + VALID_ENTRY_STATUSES, + VALID_REVIEW_SECTIONS, + AnnualReviewEntry, + AnnualReviewResult, + validate_annual_review_entry, + validate_annual_review_dict, +) + + +def _valid_entry(**kwargs) -> AnnualReviewEntry: + defaults = dict( + review_id="ANN-2026-001", + year="2026", + section="safety_policy", + reviewer="maintainer-1", + finding_count=0, + action_items_count=0, + status="completed", + notes="All safety checks passed.", + completion_date="2026-07-09", + dry_lab_only=True, + ) + defaults.update(kwargs) + return AnnualReviewEntry(**defaults) + + +class TestValidAnnualReview: + def test_valid_entry_passes(self): + result = validate_annual_review_entry(_valid_entry()) + assert result.passed is True + assert result.errors == [] + + def test_result_has_correct_review_id(self): + result = validate_annual_review_entry(_valid_entry()) + assert result.review_id == "ANN-2026-001" + + def test_result_has_correct_year(self): + result = validate_annual_review_entry(_valid_entry()) + assert result.year == "2026" + + def test_result_has_correct_section(self): + result = validate_annual_review_entry(_valid_entry()) + assert result.section == "safety_policy" + + def test_all_sections_accepted(self): + for section in VALID_REVIEW_SECTIONS: + result = validate_annual_review_entry(_valid_entry(section=section)) + assert result.passed is True, f"Expected section {section!r} to pass" + + def test_all_statuses_accepted(self): + for status in VALID_ENTRY_STATUSES: + if status == "completed": + result = validate_annual_review_entry( + _valid_entry(status=status, completion_date="2026-07-09", notes="done") + ) + else: + result = validate_annual_review_entry( + _valid_entry(status=status, completion_date="") + ) + assert result.passed is True, f"Expected status {status!r} to pass" + + def test_pending_without_completion_date_passes(self): + result = validate_annual_review_entry( + _valid_entry(status="pending", completion_date="") + ) + assert result.passed is True + + def test_dry_lab_only_is_true_in_result(self): + result = validate_annual_review_entry(_valid_entry()) + assert result.dry_lab_only is True + + def test_findings_with_action_items_passes(self): + result = validate_annual_review_entry( + _valid_entry(finding_count=2, action_items_count=2) + ) + assert result.passed is True + assert result.warnings == [] + + +class TestInvalidAnnualReview: + def test_empty_review_id_fails(self): + result = validate_annual_review_entry(_valid_entry(review_id="")) + assert result.passed is False + assert any("review_id" in e for e in result.errors) + + def test_review_id_without_ann_prefix_fails(self): + result = validate_annual_review_entry(_valid_entry(review_id="REV-2026-001")) + assert result.passed is False + assert any("ANN-" in e for e in result.errors) + + def test_invalid_year_fails(self): + result = validate_annual_review_entry(_valid_entry(year="26")) + assert result.passed is False + assert any("year" in e for e in result.errors) + + def test_invalid_section_fails(self): + result = validate_annual_review_entry(_valid_entry(section="unknown_section")) + assert result.passed is False + assert any("section" in e for e in result.errors) + + def test_empty_reviewer_fails(self): + result = validate_annual_review_entry(_valid_entry(reviewer="")) + assert result.passed is False + assert any("reviewer" in e for e in result.errors) + + def test_negative_finding_count_fails(self): + result = validate_annual_review_entry(_valid_entry(finding_count=-1)) + assert result.passed is False + assert any("finding_count" in e for e in result.errors) + + def test_negative_action_items_count_fails(self): + result = validate_annual_review_entry(_valid_entry(action_items_count=-1)) + assert result.passed is False + assert any("action_items_count" in e for e in result.errors) + + def test_invalid_status_fails(self): + result = validate_annual_review_entry(_valid_entry(status="skipped")) + assert result.passed is False + assert any("status" in e for e in result.errors) + + def test_completed_without_completion_date_fails(self): + result = validate_annual_review_entry( + _valid_entry(status="completed", completion_date="") + ) + assert result.passed is False + assert any("completion_date" in e for e in result.errors) + + def test_completed_with_bad_date_format_fails(self): + result = validate_annual_review_entry( + _valid_entry(status="completed", completion_date="09-07-2026") + ) + assert result.passed is False + assert any("completion_date" in e for e in result.errors) + + def test_dry_lab_only_false_fails(self): + result = validate_annual_review_entry(_valid_entry(dry_lab_only=False)) + assert result.passed is False + assert any("dry_lab_only" in e for e in result.errors) + + +class TestAnnualReviewWarnings: + def test_completed_no_notes_warns(self): + result = validate_annual_review_entry( + _valid_entry(status="completed", notes="", completion_date="2026-07-09") + ) + assert result.passed is True + assert any("notes" in w for w in result.warnings) + + def test_deferred_warns(self): + result = validate_annual_review_entry( + _valid_entry(status="deferred", completion_date="") + ) + assert result.passed is True + assert any("deferred" in w for w in result.warnings) + + def test_findings_without_action_items_warns(self): + result = validate_annual_review_entry( + _valid_entry(finding_count=3, action_items_count=0) + ) + assert result.passed is True + assert any("action_items" in w for w in result.warnings) + + def test_no_warning_when_findings_have_action_items(self): + result = validate_annual_review_entry( + _valid_entry(finding_count=1, action_items_count=1) + ) + assert result.passed is True + assert not any("action_items" in w for w in result.warnings) + + +class TestValidateAnnualReviewDict: + def test_valid_dict_passes(self): + d = { + "review_id": "ANN-2026-002", + "year": "2026", + "section": "benchmark_thresholds", + "reviewer": "maintainer-2", + "finding_count": 0, + "action_items_count": 0, + "status": "completed", + "notes": "Thresholds reviewed and unchanged.", + "completion_date": "2026-07-09", + "dry_lab_only": True, + } + result = validate_annual_review_dict(d) + assert result.passed is True + + def test_missing_fields_fails(self): + result = validate_annual_review_dict({"review_id": "ANN-2026-003"}) + assert result.passed is False + assert any("Missing required fields" in e for e in result.errors) + + def test_empty_dict_fails(self): + result = validate_annual_review_dict({}) + assert result.passed is False + + def test_dict_defaults_dry_lab_only_true(self): + d = { + "review_id": "ANN-2026-004", + "year": "2026", + "section": "calibration_status", + "reviewer": "maintainer-1", + "finding_count": 0, + "action_items_count": 0, + "status": "pending", + } + result = validate_annual_review_dict(d) + assert result.dry_lab_only is True diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index a823a12b..e3ea9867 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,11 +2,11 @@ import subprocess import sys -# Baseline: 3653 tests as of Loop 117 (2026-07-09) -- Phase J J9 (external advisory review process) +# Baseline: 3681 tests as of Loop 118 (2026-07-09) -- Phase J J10 (annual safety and benchmark review checklist) # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3653 +BASELINE = 3681 TOLERANCE = 0.05 # allow 5% fluctuation