feat: Phase J J10 annual safety and benchmark review checklist -- long-term trust (#810)

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check
+.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check
 
 PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
 PYTEST  := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
@@ -880,3 +880,8 @@ advisory-review-check:
 	.venv/bin/openamp-foundry advisory-review-check \
 		--review-json '{"review_id":"ADV-2026-001","review_type":"candidate_review","artifact_id":"OAMP-v0.7.6-candidates","reviewer_handle":"external-advisor-1","assigned_date":"2026-07-09","deadline_date":"2026-07-23","status":"pending"}'
 	@echo "Advisory review check complete."
+
+annual-review-check:
+	.venv/bin/openamp-foundry annual-review-check \
+		--entry-json '{"review_id":"ANN-2026-001","year":"2026","section":"safety_policy","reviewer":"maintainer-1","finding_count":0,"action_items_count":0,"status":"completed","notes":"All safety checks passed.","completion_date":"2026-07-09"}'
+	@echo "Annual review check complete."

docs/evidence/METRICS_CURRENT.md

@@ -5,7 +5,9 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
 > with this file, this file wins. Updated whenever benchmark/benchmark config changes.
 >
-> **Last updated:** 2026-07-09 (Phase J J9 — external advisory review process — v0.7.7)
+> **Last updated:** 2026-07-09 (Phase J J10 — annual safety and benchmark review checklist — v0.7.8)
+> **New in v0.7.8:** Annual safety and benchmark review checklist (J10 — Loop 118) — docs/governance/ANNUAL_REVIEW_CHECKLIST.md with 5-section structured checklist (safety_policy: 6 checks including dual-use safeguards, dry_lab_only enforcement, toxicity filters; benchmark_thresholds: 6 checks including threshold loosening guard, easy-baseline requirement, deprecation check; calibration_status: 4 checks including recalibration gate and rollback plan; governance_decisions: 4 checks including COI disclosures and rotation plan; data_governance: 3 checks including proprietary data flag). src/openamp_foundry/governance/annual_review.py with AnnualReviewEntry dataclass (10 fields: review_id, year, section, reviewer, finding_count, action_items_count, status, notes, completion_date, dry_lab_only), AnnualReviewResult dataclass (6 fields: review_id, year, section, passed, errors, warnings, dry_lab_only=True), VALID_REVIEW_SECTIONS (5: benchmark_thresholds, calibration_status, data_governance, governance_decisions, safety_policy), VALID_ENTRY_STATUSES (5: completed, deferred, in_progress, not_applicable, pending), validate_annual_review_entry() (9 checks: ANN- prefix, 4-digit year, valid section, non-empty reviewer, non-negative finding_count, non-negative action_items_count, valid status, completed requires YYYY-MM-DD completion_date, dry_lab_only must be True; completed+no-notes warns, deferred warns, findings+no-action-items warns), validate_annual_review_dict() (7 required fields guard). CLI (openamp-foundry annual-review-check) with --entry-json, --format text|json. make annual-review-check target. Long-term trust: annual review entries are now machine-validated.
+>
 > **New in v0.7.7:** External advisory review process (J9 — Loop 117) — docs/governance/EXTERNAL_ADVISORY_REVIEW_PROCESS.md with reviewer eligibility criteria, review scope table (5 review types with minimum reviewer counts), 5-step review process (prepare packet, assign+disclose, receive+log, respond to findings, close), finding severity classification (critical/major/minor/informational), limitations. src/openamp_foundry/governance/advisory_review.py with AdvisoryReview dataclass (11 fields: review_id, review_type, artifact_id, reviewer_handle, assigned_date, deadline_date, status, finding_severity, finding_summary, resolved, dry_lab_only), AdvisoryReviewResult dataclass (5 fields: review_id, review_type, passed, errors, warnings, dry_lab_only=True), VALID_REVIEW_TYPES (5: benchmark_audit, candidate_review, evidence_review, governance_review, safety_policy_review), VALID_REVIEW_STATUSES (5: assigned, completed, deferred, in_progress, pending), VALID_FINDING_SEVERITIES (4: critical, informational, major, minor), MINIMUM_REVIEWER_COUNTS (5 entries: candidate_review and safety_policy_review require 2, others 1), validate_advisory_review() (9 checks: ADV- prefix, valid review_type, non-empty artifact_id, non-empty reviewer_handle, YYYY-MM-DD dates, valid status, valid finding_severity, dry_lab_only must be True; critical+unresolved warns, completed+no-summary warns, deferred warns), validate_advisory_review_dict() (7 required fields guard). CLI (openamp-foundry advisory-review-check) with --review-json, --format text|json. make advisory-review-check target. 29 tests. Credibility: external advisory reviews now have a validated structure and documented process.
 >
 > **New in v0.7.6:** Roadmap-to-issue sync checklist (J8 — Loop 116) — docs/governance/ROADMAP_ISSUE_SYNC_CHECKLIST.md with 5-section sync checklist (roadmap items → issues, issues → roadmap, completed items, priority alignment, version consistency). src/openamp_foundry/governance/roadmap_sync.py with RoadmapSyncEntry dataclass (10 fields: item_id, phase, description, priority, sync_status, issue_number, pr_number, completed, completion_date, dry_lab_only), RoadmapSyncResult dataclass (5 fields: item_id, phase, passed, errors, warnings, dry_lab_only=True), VALID_SYNC_STATUSES (5: synced, missing_issue, orphaned_issue, stale, completed), VALID_PRIORITY_LEVELS (4: A, B, C, D), VALID_PHASES (7: E, F, G, H, I, J, K), validate_roadmap_sync_entry() (8 checks: non-empty item_id, valid phase, non-empty description, valid priority, valid sync_status, dry_lab_only must be True, completed must have completion_date, completion_date must be YYYY-MM-DD; priority A + missing_issue warns, stale warns, orphaned_issue warns, no issue_number warns), validate_roadmap_sync_dict() (5 required fields guard). CLI (openamp-foundry roadmap-sync-check) with --entry-json, --format text|json. make roadmap-sync-check target. 24 tests. Keeps strategy actionable: roadmap sync entries are now machine-validated.

docs/governance/ANNUAL_REVIEW_CHECKLIST.md

@@ -0,0 +1,56 @@
+# Annual Safety and Benchmark Review Checklist
+
+> **Scope:** Dry-lab only. No wet-lab or in-vivo claims.
+> **Cadence:** Annually, or after any major pipeline version change.
+> **Purpose:** Ensure safety policy, benchmark thresholds, calibration, governance decisions,
+> and data governance remain current and honest.
+
+---
+
+## 1. Safety Policy Review
+
+- [ ] Re-read `SAFETY.md` and confirm all dual-use safeguards remain in effect.
+- [ ] Confirm `dry_lab_only: True` is enforced on all result and report dataclasses.
+- [ ] Confirm that `evidence_level > 4` with `dry_lab_only=True` still raises an error.
+- [ ] Confirm public releases with `safety_review_status: pending` are still blocked.
+- [ ] Confirm the toxicity and hemolysis filter thresholds have not been weakened.
+- [ ] Document any safety policy changes with a new `GOV-XXX` decision log entry.
+
+## 2. Benchmark Threshold Review
+
+- [ ] Re-read `docs/evidence/BENCHMARK_GOVERNANCE.md` for threshold rationale.
+- [ ] Confirm benchmark thresholds have not been silently loosened.
+- [ ] Check whether any benchmark has been deprecated without replacement.
+- [ ] Confirm all benchmarks include an easy-baseline comparison.
+- [ ] Confirm selectivity benchmarks are still present and passing.
+- [ ] Document any threshold changes with a `GOV-XXX` decision log entry and rationale.
+
+## 3. Calibration Status Review
+
+- [ ] Run `make bench-calibration` and confirm no calibration regression.
+- [ ] Confirm the calibration decision checklist has been reviewed.
+- [ ] Confirm the recalibration gate is still active.
+- [ ] Confirm calibration rollback plan is documented and up to date.
+
+## 4. Governance Decisions Review
+
+- [ ] Run `make decision-log` and confirm all governance decisions are `active` or `superseded` with rationale.
+- [ ] Confirm no governance decision is in `under_review` status for more than 30 days without action.
+- [ ] Confirm COI disclosures are filed for all maintainers and external advisors.
+- [ ] Confirm maintainer rotation plan is up to date (`make rotation-plan-check`).
+
+## 5. Data Governance Review
+
+- [ ] Confirm `DATA_LICENSE_NOTICE.md` is current.
+- [ ] Confirm no proprietary data has entered the pipeline without a `contact_required` license flag.
+- [ ] Confirm all external data sources are documented in the data governance log.
+
+## 6. Action Items
+
+After completing each section, record findings using `make annual-review-check`.
+Any `finding_count > 0` must have a corresponding `action_items_count > 0`.
+Critical findings halt the next candidate release until resolved.
+
+---
+
+*This checklist is machine-validated. See `src/openamp_foundry/governance/annual_review.py`.*

docs/research/ROADMAP.md

@@ -1,5 +1,33 @@
 # Roadmap
 
+## v0.7.8 — Loop 118: Phase J J10 — Annual Safety and Benchmark Review Checklist ✓ (2026-07-09)
+
+`docs/governance/ANNUAL_REVIEW_CHECKLIST.md` with 5-section structured annual
+review checklist (safety_policy: 6 checks covering dual-use safeguards,
+dry_lab_only enforcement, toxicity/hemolysis filter thresholds, evidence_level
+guard; benchmark_thresholds: 6 checks covering threshold loosening guard,
+easy-baseline requirement, selectivity benchmarks, deprecation check;
+calibration_status: 4 checks covering recalibration gate, decision checklist,
+rollback plan; governance_decisions: 4 checks covering active decisions,
+COI disclosures, maintainer rotation; data_governance: 3 checks covering
+proprietary data license flags, external source documentation).
+
+`src/openamp_foundry/governance/annual_review.py` with `AnnualReviewEntry`
+dataclass (10 fields: review_id, year, section, reviewer, finding_count,
+action_items_count, status, notes, completion_date, dry_lab_only),
+`AnnualReviewResult` dataclass (6 fields, dry_lab_only=True),
+`VALID_REVIEW_SECTIONS` (5: benchmark_thresholds, calibration_status,
+data_governance, governance_decisions, safety_policy),
+`VALID_ENTRY_STATUSES` (5: completed, deferred, in_progress, not_applicable,
+pending), `validate_annual_review_entry()` (9 checks: ANN- prefix, 4-digit year,
+valid section, non-empty reviewer, non-negative finding_count,
+non-negative action_items_count, valid status, completed requires YYYY-MM-DD
+completion_date, dry_lab_only must be True; completed+no-notes warns, deferred
+warns, findings+no-action-items warns), `validate_annual_review_dict()` (7
+required fields guard). CLI (`openamp-foundry annual-review-check`) with
+`--entry-json`, `--format text|json`. `make annual-review-check` target.
+Long-term trust: annual review entries are now machine-validated.
+
 ## v0.7.7 — Loop 117: Phase J J9 — External Advisory Review Process ✓ (2026-07-09)
 
 `docs/governance/EXTERNAL_ADVISORY_REVIEW_PROCESS.md` with reviewer eligibility

src/openamp_foundry/cli/commands/reports.py

@@ -2783,3 +2783,37 @@ def _run_roadmap_sync_check(args: argparse.Namespace) -> int:
             print("  Roadmap sync entry validated. Strategy is actionable.")
 
     return 0 if result.passed else 3
+
+
+def _run_annual_review_check(args: argparse.Namespace) -> int:
+    """Validate an annual review checklist entry."""
+    import json as _json
+    from openamp_foundry.governance.annual_review import validate_annual_review_dict
+
+    try:
+        d = _json.loads(args.entry_json)
+    except _json.JSONDecodeError as exc:
+        print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"}))
+        return 2
+
+    if not isinstance(d, dict):
+        print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"}))
+        return 2
+
+    result = validate_annual_review_dict(d)
+    output_format = getattr(args, "format", "text")
+
+    if output_format == "json":
+        import dataclasses
+        print(_json.dumps(dataclasses.asdict(result), indent=2))
+    else:
+        status = "PASS" if result.passed else "FAIL"
+        print(f"Annual review {result.review_id} ({result.section} / {result.year}): {status}")
+        for e in result.errors:
+            print(f"  ERROR: {e}")
+        for w in result.warnings:
+            print(f"  WARN:  {w}")
+        if result.passed:
+            print("  Annual review entry validated. Long-term trust maintained.")
+
+    return 0 if result.passed else 3

src/openamp_foundry/cli/main.py

@@ -15,7 +15,7 @@
 from openamp_foundry.cli.commands.reports import _run_security_report_check
 from openamp_foundry.cli.commands.reports import _run_citation_check
 from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check
-from openamp_foundry.cli.commands.reports import _run_advisory_review_check
+from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check
 
 import argparse
 import json
@@ -1999,6 +1999,24 @@ def build_parser() -> argparse.ArgumentParser:
         help="Output format (default: text).",
     )
 
+    # ── Annual review check (Phase J J10) ───────────────────────────
+    anrc = sub.add_parser(
+        "annual-review-check",
+        help=(
+            "Validate an annual safety and benchmark review checklist entry. "
+            "Dry-lab only."
+        ),
+    )
+    anrc.add_argument(
+        "--entry-json", type=str, required=True,
+        help="JSON string of an AnnualReviewEntry dict (required).",
+    )
+    anrc.add_argument(
+        "--format", type=str, default="text",
+        choices=["text", "json"],
+        help="Output format (default: text).",
+    )
+
     # ── Simulation-result confidence interval report (H7) ────────────
     sicr = sub.add_parser(
         "simulation-ci-report",
@@ -2264,6 +2282,9 @@ def main(argv: list[str] | None = None) -> int:
     if args.command == "advisory-review-check":
         return _run_advisory_review_check(args)
 
+    if args.command == "annual-review-check":
+        return _run_annual_review_check(args)
+
     parser.error("unknown command")
     return 2