From c7c99afdf0921776eaac4c3d8eb7185838a89ec1 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Wed, 8 Jul 2026 14:43:12 +0700 Subject: [PATCH] Add 10 decision trees, walkthroughs, and runbooks Closes #529, Closes #530, Closes #531, Closes #532, Closes #533, Closes #535, Closes #536, Closes #537, Closes #538, Closes #539 --- .../DECISION_TREE_BENCHMARK_CHANGE.md | 11 ++++++++++ .../DECISION_TREE_EVIDENCE_DOWNGRADE.md | 12 +++++++++++ .../DECISION_TREE_FIRST_CONTRIBUTION.md | 12 +++++++++++ .../evidence/DECISION_TREE_OPENING_DOCS_PR.md | 12 +++++++++++ docs/evidence/DECISION_TREE_PUBLIC_SHARING.md | 12 +++++++++++ docs/evidence/GUIDE_RELEASE_READINESS.md | 16 +++++++++++++++ docs/evidence/RUNBOOK_BROKEN_DOCS_LINKS.md | 9 +++++++++ docs/evidence/RUNBOOK_STALE_ARTIFACTS.md | 10 ++++++++++ docs/evidence/RUNBOOK_UNSAFE_CLAIMS.md | 10 ++++++++++ .../evidence/WALKTHROUGH_CLAIM_TO_EVIDENCE.md | 16 +++++++++++++++ tests/test_decision_trees.py | 20 +++++++++++++++++++ 11 files changed, 140 insertions(+) create mode 100644 docs/evidence/DECISION_TREE_BENCHMARK_CHANGE.md create mode 100644 docs/evidence/DECISION_TREE_EVIDENCE_DOWNGRADE.md create mode 100644 docs/evidence/DECISION_TREE_FIRST_CONTRIBUTION.md create mode 100644 docs/evidence/DECISION_TREE_OPENING_DOCS_PR.md create mode 100644 docs/evidence/DECISION_TREE_PUBLIC_SHARING.md create mode 100644 docs/evidence/GUIDE_RELEASE_READINESS.md create mode 100644 docs/evidence/RUNBOOK_BROKEN_DOCS_LINKS.md create mode 100644 docs/evidence/RUNBOOK_STALE_ARTIFACTS.md create mode 100644 docs/evidence/RUNBOOK_UNSAFE_CLAIMS.md create mode 100644 docs/evidence/WALKTHROUGH_CLAIM_TO_EVIDENCE.md create mode 100644 tests/test_decision_trees.py diff --git a/docs/evidence/DECISION_TREE_BENCHMARK_CHANGE.md b/docs/evidence/DECISION_TREE_BENCHMARK_CHANGE.md new file mode 100644 index 00000000..3531a228 --- /dev/null +++ b/docs/evidence/DECISION_TREE_BENCHMARK_CHANGE.md @@ -0,0 +1,11 @@ +# Decision Tree: Benchmark Change Review + +```mermaid +flowchart TD + A[Benchmark change proposed] --> B{Does it change AUROC?} + B -->|Yes, improvement| C[Verify no overfitting] + B -->|Yes, regression| D[Is regression > 0.02?] + D -->|Yes| E[Block — investigate] + D -->|No| F[Accept — document tradeoff] + B -->|No change| G[Verify benchmark still runs] +``` diff --git a/docs/evidence/DECISION_TREE_EVIDENCE_DOWNGRADE.md b/docs/evidence/DECISION_TREE_EVIDENCE_DOWNGRADE.md new file mode 100644 index 00000000..8f9e135d --- /dev/null +++ b/docs/evidence/DECISION_TREE_EVIDENCE_DOWNGRADE.md @@ -0,0 +1,12 @@ +# Decision Tree: Evidence Downgrade + +```mermaid +flowchart TD + A[Evidence quality concern] --> B{Is the evidence reproducible?} + B -->|No| C[Downgrade 1 level] + B -->|Yes| D{Are the caveats documented?} + D -->|No| E[Downgrade 1 level] + D -->|Yes| F{Is the data source reliable?} + F -->|No| G[Downgrade 1 level] + F -->|Yes| H[Maintain current level] +``` diff --git a/docs/evidence/DECISION_TREE_FIRST_CONTRIBUTION.md b/docs/evidence/DECISION_TREE_FIRST_CONTRIBUTION.md new file mode 100644 index 00000000..f914f590 --- /dev/null +++ b/docs/evidence/DECISION_TREE_FIRST_CONTRIBUTION.md @@ -0,0 +1,12 @@ +# Decision Tree: Choosing First Contribution + +```mermaid +flowchart TD + A[Want to contribute?] --> B{Can you run Python?} + B -->|Yes| C{Can you write tests?} + B -->|No| D[Start with docs-only tasks] + C -->|Yes| E[Pick a test issue] + C -->|No| F{Can you write docs?} + F -->|Yes| G[Pick a docs issue] + F -->|No| H[Start with CONTRIBUTOR_CURRICULUM.md] +``` diff --git a/docs/evidence/DECISION_TREE_OPENING_DOCS_PR.md b/docs/evidence/DECISION_TREE_OPENING_DOCS_PR.md new file mode 100644 index 00000000..e7e6afb6 --- /dev/null +++ b/docs/evidence/DECISION_TREE_OPENING_DOCS_PR.md @@ -0,0 +1,12 @@ +# Decision Tree: Opening a Documentation PR + +```mermaid +flowchart TD + A[Change needed in docs] --> B{Is it a typo or formatting?} + B -->|Yes| C[Direct commit or minor PR] + B -->|No| D{Is it a policy change?} + D -->|Yes| E[Full PR + maintainer review] + D -->|No| F{Is it a new document?} + F -->|Yes| G[PR + link from PROJECT_INDEX.md] + F -->|No| H[Standard PR with review] +``` diff --git a/docs/evidence/DECISION_TREE_PUBLIC_SHARING.md b/docs/evidence/DECISION_TREE_PUBLIC_SHARING.md new file mode 100644 index 00000000..395c7fc6 --- /dev/null +++ b/docs/evidence/DECISION_TREE_PUBLIC_SHARING.md @@ -0,0 +1,12 @@ +# Decision Tree: Public Artifact Sharing + +```mermaid +flowchart TD + A[Share artifact publicly?] --> B{Contains sensitive data?} + B -->|Yes| C[Redact or don't share] + B -->|No| D{Is it a release artifact?} + D -->|Yes| E[Follow PUBLICATION_PACK.md] + D -->|No| F{Is it a benchmark result?} + F -->|Yes| G[Include caveats] + F -->|No| H[Share with standard disclaimer] +``` diff --git a/docs/evidence/GUIDE_RELEASE_READINESS.md b/docs/evidence/GUIDE_RELEASE_READINESS.md new file mode 100644 index 00000000..6d57f384 --- /dev/null +++ b/docs/evidence/GUIDE_RELEASE_READINESS.md @@ -0,0 +1,16 @@ +# Decision Guide: Release Readiness + +Before releasing, verify: + +- [ ] `make full-reproducibility-report` succeeds +- [ ] `python scripts/check_doc_links.py` — 0 broken links +- [ ] `python scripts/safety/check_claims.py` — 0 overclaiming +- [ ] `python3 -m pytest -q` — all tests pass +- [ ] METRICS_CURRENT.md is up to date +- [ ] PUBLICATION_PACK.md checklist is complete +- [ ] Decision log is current + +## Decision +- All checks pass → Release candidate +- Minor issues → Fix before release +- Major issues → Block release diff --git a/docs/evidence/RUNBOOK_BROKEN_DOCS_LINKS.md b/docs/evidence/RUNBOOK_BROKEN_DOCS_LINKS.md new file mode 100644 index 00000000..2d3ffcc9 --- /dev/null +++ b/docs/evidence/RUNBOOK_BROKEN_DOCS_LINKS.md @@ -0,0 +1,9 @@ +# Maintainer Runbook: Broken Docs Links + +When doc links are broken: + +1. Run `python scripts/check_doc_links.py` to find all broken links. +2. For each broken link, determine the correct target path. +3. Update the link in the source document. +4. If the target document was moved, add a redirect or update all references. +5. Re-run the check to confirm 0 broken links. diff --git a/docs/evidence/RUNBOOK_STALE_ARTIFACTS.md b/docs/evidence/RUNBOOK_STALE_ARTIFACTS.md new file mode 100644 index 00000000..5137eda1 --- /dev/null +++ b/docs/evidence/RUNBOOK_STALE_ARTIFACTS.md @@ -0,0 +1,10 @@ +# Maintainer Runbook: Stale Artifacts + +When an artifact is stale: + +1. Identify what's stale and why. +2. If it's a benchmark output, regenerate with `make bench-*`. +3. If it's a schema, update it to match current code. +4. If it's a config, update or document the discrepancy. +5. Update references in docs. +6. Add a test to prevent future staleness. diff --git a/docs/evidence/RUNBOOK_UNSAFE_CLAIMS.md b/docs/evidence/RUNBOOK_UNSAFE_CLAIMS.md new file mode 100644 index 00000000..31a79c12 --- /dev/null +++ b/docs/evidence/RUNBOOK_UNSAFE_CLAIMS.md @@ -0,0 +1,10 @@ +# Maintainer Runbook: Unsafe Claim Wording + +When unsafe claim wording is found: + +1. Run `python scripts/safety/check_claims.py` to see all flagged phrases. +2. Review each flagged phrase in context. +3. Rewrite overclaiming phrases using project-allowed terms. +4. If the claim is in a public-facing document, prioritize the fix. +5. Re-run the check to confirm the fix. +6. If the author disagrees, escalate to maintainers. diff --git a/docs/evidence/WALKTHROUGH_CLAIM_TO_EVIDENCE.md b/docs/evidence/WALKTHROUGH_CLAIM_TO_EVIDENCE.md new file mode 100644 index 00000000..925c86b2 --- /dev/null +++ b/docs/evidence/WALKTHROUGH_CLAIM_TO_EVIDENCE.md @@ -0,0 +1,16 @@ +# Guided Walkthrough: Claim-to-Evidence Matrices + +How to map a claim to its supporting evidence. + +## Steps +1. State the claim clearly — what exactly is being asserted? +2. Identify the proof level required (see PROOF_LADDER.md). +3. Find the evidence that supports the claim. +4. Check if the evidence level matches the claim strength. +5. If the claim is stronger than the evidence, weaken the claim or gather more evidence. + +## Example +| Claim | Required Level | Available Evidence | Match? | +|-------|:--------------:|-------------------|:------:| +| "Candidate X is predicted antimicrobial" | 2 (baseline score) | Ensemble score 0.85 | ✅ | +| "Candidate X is a drug candidate" | 5 (lab assay) | No wet-lab data | ❌ | diff --git a/tests/test_decision_trees.py b/tests/test_decision_trees.py new file mode 100644 index 00000000..cdb66399 --- /dev/null +++ b/tests/test_decision_trees.py @@ -0,0 +1,20 @@ +"""Tests for batch of 10 docs.""" +from pathlib import Path + +FILES = [ + "WALKTHROUGH_CLAIM_TO_EVIDENCE.md", + "DECISION_TREE_FIRST_CONTRIBUTION.md", + "DECISION_TREE_OPENING_DOCS_PR.md", + "DECISION_TREE_BENCHMARK_CHANGE.md", + "GUIDE_RELEASE_READINESS.md", + "DECISION_TREE_PUBLIC_SHARING.md", + "DECISION_TREE_EVIDENCE_DOWNGRADE.md", + "RUNBOOK_STALE_ARTIFACTS.md", + "RUNBOOK_BROKEN_DOCS_LINKS.md", + "RUNBOOK_UNSAFE_CLAIMS.md", +] + + +def test_all_exist(): + for f in FILES: + assert Path("docs/evidence", f).exists(), f"Missing: {f}"