From 6948c989bc0cb8c5c7b830ebe2e5648e9d9d4331 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Tue, 7 Jul 2026 11:38:20 +0700 Subject: [PATCH] Add 5 metric guides Closes #654, Closes #655, Closes #656, Closes #657, Closes #658 - Onboarding success, safety clarity, artifact explainability, benchmark explainability, review readiness --- .../METRIC_GUIDE_ARTIFACT_EXPLAINABILITY.md | 11 ++++++++++ .../METRIC_GUIDE_BENCHMARK_EXPLAINABILITY.md | 11 ++++++++++ .../METRIC_GUIDE_ONBOARDING_SUCCESS.md | 11 ++++++++++ .../evidence/METRIC_GUIDE_REVIEW_READINESS.md | 16 ++++++++++++++ docs/evidence/METRIC_GUIDE_SAFETY_CLARITY.md | 11 ++++++++++ tests/test_metric_guides.py | 21 +++++++++++++++++++ 6 files changed, 81 insertions(+) create mode 100644 docs/evidence/METRIC_GUIDE_ARTIFACT_EXPLAINABILITY.md create mode 100644 docs/evidence/METRIC_GUIDE_BENCHMARK_EXPLAINABILITY.md create mode 100644 docs/evidence/METRIC_GUIDE_ONBOARDING_SUCCESS.md create mode 100644 docs/evidence/METRIC_GUIDE_REVIEW_READINESS.md create mode 100644 docs/evidence/METRIC_GUIDE_SAFETY_CLARITY.md create mode 100644 tests/test_metric_guides.py diff --git a/docs/evidence/METRIC_GUIDE_ARTIFACT_EXPLAINABILITY.md b/docs/evidence/METRIC_GUIDE_ARTIFACT_EXPLAINABILITY.md new file mode 100644 index 00000000..0aef4c8f --- /dev/null +++ b/docs/evidence/METRIC_GUIDE_ARTIFACT_EXPLAINABILITY.md @@ -0,0 +1,11 @@ +# Metric Guide: Artifact Explainability + +**Question:** Can a reviewer understand what a given artifact means and how it was produced? + +**What to track:** Whether each artifact type has a corresponding concept card or documentation section. + +**Target:** Every artifact type in ARTIFACT_MAP.md has a concept card. + +**What to measure:** Count of artifact types with concept cards divided by total artifact types. + +**What good looks like:** 100% coverage. diff --git a/docs/evidence/METRIC_GUIDE_BENCHMARK_EXPLAINABILITY.md b/docs/evidence/METRIC_GUIDE_BENCHMARK_EXPLAINABILITY.md new file mode 100644 index 00000000..9f5d6472 --- /dev/null +++ b/docs/evidence/METRIC_GUIDE_BENCHMARK_EXPLAINABILITY.md @@ -0,0 +1,11 @@ +# Metric Guide: Benchmark Explainability + +**Question:** Can a reader understand what each benchmark measures and its limitations? + +**What to track:** Whether each benchmark in BENCHMARKING.md has a caveat section. + +**Target:** Every benchmark has at least one documented caveat. + +**What to measure:** Ratio of benchmarks with caveats to total benchmarks. + +**What good looks like:** A reader can find any benchmark and immediately see what it does NOT prove. diff --git a/docs/evidence/METRIC_GUIDE_ONBOARDING_SUCCESS.md b/docs/evidence/METRIC_GUIDE_ONBOARDING_SUCCESS.md new file mode 100644 index 00000000..945fff41 --- /dev/null +++ b/docs/evidence/METRIC_GUIDE_ONBOARDING_SUCCESS.md @@ -0,0 +1,11 @@ +# Metric Guide: Onboarding Success + +**Question:** How quickly can a new contributor go from zero to running the demo? + +**What to track:** Time from `git clone` to `make demo` completing successfully. + +**Target:** < 15 minutes per QUICKSTART.md. + +**How to measure:** Run through QUICKSTART.md with a stopwatch. + +**What good looks like:** A contributor with Python installed can complete the full path in one sitting without asking for help. diff --git a/docs/evidence/METRIC_GUIDE_REVIEW_READINESS.md b/docs/evidence/METRIC_GUIDE_REVIEW_READINESS.md new file mode 100644 index 00000000..790488fc --- /dev/null +++ b/docs/evidence/METRIC_GUIDE_REVIEW_READINESS.md @@ -0,0 +1,16 @@ +# Metric Guide: Review Readiness + +**Question:** Is the project ready for external review? + +**What to track:** Whether all review artifacts exist and are up to date. + +**Artifacts to check:** +- docs/review/EXPERT_REVIEW_PACK.md +- docs/review/RELEASE_DRY_RUN_GUIDE.md +- docs/review/PUBLICATION_PACK.md +- docs/evidence/LIMITATIONS_OVERVIEW.md +- .github/ISSUE_TEMPLATE/expert_review.yml + +**Target:** All 5 artifacts exist and pass claim check. + +**What good looks like:** A reviewer can be pointed to the review pack and have everything they need. diff --git a/docs/evidence/METRIC_GUIDE_SAFETY_CLARITY.md b/docs/evidence/METRIC_GUIDE_SAFETY_CLARITY.md new file mode 100644 index 00000000..7f4a4d30 --- /dev/null +++ b/docs/evidence/METRIC_GUIDE_SAFETY_CLARITY.md @@ -0,0 +1,11 @@ +# Metric Guide: Safety Clarity + +**Question:** Can a reviewer find the safety policy and understand what it permits? + +**What to track:** Time to find and understand the safety policy. + +**Target:** < 5 minutes. + +**What to measure:** A new reader can find SAFETY.md, identify allowed and forbidden uses, and state the dry-lab boundary within 5 minutes. + +**What good looks like:** Reviewer can quote the relevant policy section without assistance. diff --git a/tests/test_metric_guides.py b/tests/test_metric_guides.py new file mode 100644 index 00000000..d987289f --- /dev/null +++ b/tests/test_metric_guides.py @@ -0,0 +1,21 @@ +"""Tests for metric guides.""" +from pathlib import Path + +GUIDES = [ + "METRIC_GUIDE_ONBOARDING_SUCCESS.md", + "METRIC_GUIDE_SAFETY_CLARITY.md", + "METRIC_GUIDE_ARTIFACT_EXPLAINABILITY.md", + "METRIC_GUIDE_BENCHMARK_EXPLAINABILITY.md", + "METRIC_GUIDE_REVIEW_READINESS.md", +] + + +def test_all_exist(): + for g in GUIDES: + assert Path(f"docs/evidence/{g}").exists(), f"Missing: {g}" + + +def test_each_has_question(): + for g in GUIDES: + text = Path(f"docs/evidence/{g}").read_text() + assert "Question" in text or "question" in text