diff --git a/docs/evidence/GUIDE_PR_SAFETY_CHECK.md b/docs/evidence/GUIDE_PR_SAFETY_CHECK.md new file mode 100644 index 00000000..2d82be4d --- /dev/null +++ b/docs/evidence/GUIDE_PR_SAFETY_CHECK.md @@ -0,0 +1,21 @@ +# PR Template Check for Safety-Sensitive Files + +PRs that modify safety-sensitive files require additional checks. + +## Safety-Sensitive Files +- `SAFETY.md` +- `RESPONSIBLE_USE.md` +- `docs/trust/*` +- `AGENTS.md` +- `configs/recalibration_policy.yaml` +- `configs/pipeline.yaml` + +## Required Checks +- [ ] Change has been reviewed by a safety reviewer. +- [ ] No overclaiming language introduced (run `check_claims.py`). +- [ ] Safety implications documented in PR description. +- [ ] All existing safety tests pass. + +## Enforcement +- PR template should include a section for safety-sensitive changes. +- CI should flag PRs that touch safety-sensitive files without a safety reviewer. diff --git a/docs/evidence/GUIDE_PUBLIC_SUMMARY_GENERATOR.md b/docs/evidence/GUIDE_PUBLIC_SUMMARY_GENERATOR.md new file mode 100644 index 00000000..584363d2 --- /dev/null +++ b/docs/evidence/GUIDE_PUBLIC_SUMMARY_GENERATOR.md @@ -0,0 +1,34 @@ +# Public Summary Generator for Review Packets + +Generates a public summary from review packet contents. + +## Inputs +- Evidence certificates for selected candidates +- LIMITATIONS_OVERVIEW.md +- METRICS_CURRENT.md (summary) +- The specific claim being reviewed + +## Output Format +```markdown +# Summary: [Claim] + +## What Was Done +One paragraph description. + +## Key Results +- Metric: value +- Key finding: statement + +## Limitations +- Known caveat 1 +- Known caveat 2 + +## Next Steps +- What happens next. +``` + +## Rules +- The summary must not overclaim. +- Run `check_claims.py` on the generated summary. +- Limitations should be as prominent as results. +- The summary should be understandable by a non-specialist. diff --git a/docs/evidence/GUIDE_REVIEW_PACKET_DIFF.md b/docs/evidence/GUIDE_REVIEW_PACKET_DIFF.md new file mode 100644 index 00000000..61f729fc --- /dev/null +++ b/docs/evidence/GUIDE_REVIEW_PACKET_DIFF.md @@ -0,0 +1,22 @@ +# Review Packet Diff Report + +Reports differences between review packet versions. + +## Format +```diff ++ Added: SEED-020_VAR_004 (new candidate) +- Removed: SEED-015_VAR_002 (deprioritized) +~ Changed: LIMITATIONS_OVERVIEW.md (updated simulation section) +``` + +## When to Generate +- When a review packet is updated after initial review. +- When new candidates are added to the panel. +- When benchmark results change significantly. +- Before sending the final packet to reviewers. + +## Rules +- Changes should be highlighted, not hidden. +- Reviewers should be notified of significant changes. +- The diff should be included in the review packet cover note. +- If changes are extensive, consider a new review cycle. diff --git a/docs/evidence/POLICY_SLOW_TEST.md b/docs/evidence/POLICY_SLOW_TEST.md new file mode 100644 index 00000000..470ebf0a --- /dev/null +++ b/docs/evidence/POLICY_SLOW_TEST.md @@ -0,0 +1,25 @@ +# Slow-Test Marker Policy + +How to mark and handle slow tests. + +## Marking +```python +@pytest.mark.slow +def test_expensive_operation(): + ... +``` + +## Running +```bash +# Skip slow tests +pytest -m "not slow" + +# Only slow tests +pytest -m slow +``` + +## Rules +- Tests that take > 5s should be marked as slow. +- CI should run both slow and fast tests. +- Local development should default to skipping slow tests. +- Slow tests should be reviewed for optimization opportunities. diff --git a/docs/evidence/REPORT_TEST_RUNTIME_BUDGET.md b/docs/evidence/REPORT_TEST_RUNTIME_BUDGET.md new file mode 100644 index 00000000..f1665c5a --- /dev/null +++ b/docs/evidence/REPORT_TEST_RUNTIME_BUDGET.md @@ -0,0 +1,20 @@ +# Test Runtime Budget Report + +Reports how long tests take to run and flags slow tests. + +## Current Status +Full test suite: ~80 seconds (varies by machine) + +## Budget +| Test Category | Budget | Current | +|:-------------|:------:|:-------:| +| Unit tests | < 30s | ✅ | +| CLI tests | < 30s | ✅ | +| Integration tests | < 30s | ✅ | +| Full suite | < 120s | ✅ | + +## Rules +- If a test consistently takes > 5s, mark it as slow. +- Slow tests should be optimized or moved to a separate CI job. +- Test runtime should be tracked in CI output. +- If the full suite exceeds 120s, investigate optimization. diff --git a/docs/evidence/TAXONOMY_REVIEWER_ROLES.md b/docs/evidence/TAXONOMY_REVIEWER_ROLES.md new file mode 100644 index 00000000..59623c0b --- /dev/null +++ b/docs/evidence/TAXONOMY_REVIEWER_ROLES.md @@ -0,0 +1,17 @@ +# Reviewer Role Taxonomy + +Defines reviewer roles and their responsibilities. + +| Role | Can Approve | Required For | Review Depth | +|------|:-----------:|--------------|:------------:| +| Contributor | No | All PRs | Comment only | +| Reviewer | Low-risk | Medium-risk PRs | Functional | +| Maintainer | High-risk | High-risk PRs | Deep | +| Safety reviewer | Safety changes | Safety-sensitive files | Full audit | +| Project lead | All | Final approval | Final | + +## Assignment +- Reviewers should be assigned based on expertise. +- Safety changes always need a safety reviewer. +- High-risk changes need at least 2 reviewers. +- The project lead has final authority on disputes. diff --git a/docs/evidence/TEMPLATE_ISSUE_BENCHMARK_CHANGE.md b/docs/evidence/TEMPLATE_ISSUE_BENCHMARK_CHANGE.md new file mode 100644 index 00000000..d330c2ca --- /dev/null +++ b/docs/evidence/TEMPLATE_ISSUE_BENCHMARK_CHANGE.md @@ -0,0 +1,31 @@ +# Issue Template for Benchmark Changes + +Template for issues that propose benchmark changes. + +```markdown +## Summary +Brief description of the benchmark change. + +## Current Benchmark +Link to current benchmark card and metrics. + +## Proposed Change +What benchmark is being added, modified, or removed. + +## Rationale +Why this change is needed. + +## Metrics +What metrics will be affected. + +## Gate Impact +Will this change affect the benchmark regression gate? + +## Related +- Link to related PRs or discussions. +``` + +## Rules +- Benchmark changes affecting the gate require maintainer review. +- New benchmarks must include documented caveats. +- Benchmark changes should be discussed before implementation. diff --git a/docs/evidence/TEMPLATE_ISSUE_SCHEMA_CHANGE.md b/docs/evidence/TEMPLATE_ISSUE_SCHEMA_CHANGE.md new file mode 100644 index 00000000..9cfa4751 --- /dev/null +++ b/docs/evidence/TEMPLATE_ISSUE_SCHEMA_CHANGE.md @@ -0,0 +1,32 @@ +# Issue Template for Artifact Schema Changes + +Template for issues that propose schema changes. + +```markdown +## Summary +Brief description of the proposed schema change. + +## Current Schema +Link to current schema file and relevant fields. + +## Proposed Change +What fields are being added, removed, or changed. + +## Rationale +Why this change is needed. + +## Backward Compatibility +- Is this change backward-compatible? +- If not, what is the migration plan? + +## Affected Artifacts +List all artifacts that use this schema. + +## Related +- Link to related PRs or discussions. +``` + +## Rules +- Schema changes require maintainer review. +- Breaking schema changes require a migration plan. +- New fields should be optional when possible. diff --git a/tests/test_proof_ladder_validator.py b/tests/test_proof_ladder_validator.py new file mode 100644 index 00000000..fdc1eb35 --- /dev/null +++ b/tests/test_proof_ladder_validator.py @@ -0,0 +1,27 @@ +"""Test proof ladder field validation in artifacts.""" +from pathlib import Path + + +def test_evidence_certificate_has_proof_fields(): + """Evidence certificates should contain fields relevant to proof level.""" + import json + schema = json.loads(Path("schemas/candidate.schema.json").read_text()) + props = str(schema.get("properties", {})) + assert "scores" in props, "Missing scores in evidence schema" + assert "candidate" in props or "candidate_id" in props + + +def test_lab_result_has_proof_fields(): + import json + schema = json.loads(Path("schemas/lab_result.schema.json").read_text()) + required = schema.get("required", []) + for field in ["candidate_id", "assay_type", "result_value"]: + assert field in required + + +def test_pipeline_version_tracked(): + """Pipeline version should be tracked in key artifacts.""" + import json + for schema_name in ["candidate.schema.json", "run_manifest.schema.json"]: + schema = json.loads(Path(f"schemas/{schema_name}").read_text()) + assert "version" in str(schema) or "pipeline" in str(schema).lower() diff --git a/tests/test_release_status_enum.py b/tests/test_release_status_enum.py new file mode 100644 index 00000000..31e79718 --- /dev/null +++ b/tests/test_release_status_enum.py @@ -0,0 +1,25 @@ +"""Test release status enum values against registries.""" +from pathlib import Path +import json + + +def test_exit_code_registry_has_expected_codes(): + registry = json.loads(Path("configs/exit_code_registry.json").read_text()) + codes = registry.get("codes", {}) + for code_str in ["0", "1", "2", "3"]: + assert code_str in codes, f"Missing exit code {code_str}" + assert "meaning" in codes[code_str] + + +def test_exit_code_meanings_are_consistent(): + registry = json.loads(Path("configs/exit_code_registry.json").read_text()) + codes = registry.get("codes", {}) + meanings = {v["meaning"] for v in codes.values()} + assert len(meanings) == len(codes), "Duplicate meanings found" + + +def test_decision_log_enum_values(): + schema = json.loads(Path("schemas/decision_log.schema.json").read_text()) + decision_types = schema["properties"]["decision_type"]["enum"] + for dt in ["approved", "rejected", "deferred"]: + assert dt in schema["properties"]["decision"]["enum"]