From f046ea1ac477a1ee33dc4d7eaf02562d0b8b131c Mon Sep 17 00:00:00 2001 From: OpenCode Date: Tue, 7 Jul 2026 11:30:07 +0700 Subject: [PATCH] Add 5 concept cards Closes #611, Closes #612, Closes #613, Closes #614, Closes #616 - CONCEPT_CARD_PROOF_LADDER.md -- evidence levels 0-6 - CONCEPT_CARD_ARTIFACT.md -- artifact types and rules - CONCEPT_CARD_SCHEMA.md -- schema formats and rules - CONCEPT_CARD_MANIFEST.md -- run manifest required fields - CONCEPT_CARD_BENCHMARK_CAVEAT.md -- benchmark limitations - 3 shared tests --- docs/evidence/CONCEPT_CARD_ARTIFACT.md | 25 +++++++++++++++ .../evidence/CONCEPT_CARD_BENCHMARK_CAVEAT.md | 26 ++++++++++++++++ docs/evidence/CONCEPT_CARD_MANIFEST.md | 25 +++++++++++++++ docs/evidence/CONCEPT_CARD_PROOF_LADDER.md | 25 +++++++++++++++ docs/evidence/CONCEPT_CARD_SCHEMA.md | 22 +++++++++++++ docs/evidence/RELEASE_STATUS_CONCEPT_CARD.md | 31 +++++++++++++++++++ tests/test_concept_cards.py | 27 ++++++++++++++++ 7 files changed, 181 insertions(+) create mode 100644 docs/evidence/CONCEPT_CARD_ARTIFACT.md create mode 100644 docs/evidence/CONCEPT_CARD_BENCHMARK_CAVEAT.md create mode 100644 docs/evidence/CONCEPT_CARD_MANIFEST.md create mode 100644 docs/evidence/CONCEPT_CARD_PROOF_LADDER.md create mode 100644 docs/evidence/CONCEPT_CARD_SCHEMA.md create mode 100644 docs/evidence/RELEASE_STATUS_CONCEPT_CARD.md create mode 100644 tests/test_concept_cards.py diff --git a/docs/evidence/CONCEPT_CARD_ARTIFACT.md b/docs/evidence/CONCEPT_CARD_ARTIFACT.md new file mode 100644 index 00000000..d5c57d5d --- /dev/null +++ b/docs/evidence/CONCEPT_CARD_ARTIFACT.md @@ -0,0 +1,25 @@ +# Artifact — Concept Card + +An artifact is any file generated by the pipeline that carries evidence, +a score, or a decision record. + +## Types + +| Type | Examples | Must be versioned? | +|------|----------|:------------------:| +| Input | Candidate CSV, config YAML | Yes | +| Intermediate | Ranked JSONL, evidence certs | Yes | +| Report | Batch report, benchmark output | Yes | +| Manifest | Run manifest, batch manifest | Yes | +| Review | Review request, decision log | Yes | + +## Rules + +- Every artifact should have a defined schema or format. +- Artifacts that affect decisions must be reproducible. +- Temporary artifacts (caches, logs) are excluded. + +## Related + +- `docs/evidence/ARTIFACT_MAP.md` — complete artifact catalog +- `docs/engineering/RUN_MANIFEST_STANDARD.md` — manifest specification diff --git a/docs/evidence/CONCEPT_CARD_BENCHMARK_CAVEAT.md b/docs/evidence/CONCEPT_CARD_BENCHMARK_CAVEAT.md new file mode 100644 index 00000000..63d6f205 --- /dev/null +++ b/docs/evidence/CONCEPT_CARD_BENCHMARK_CAVEAT.md @@ -0,0 +1,26 @@ +# Benchmark Caveat — Concept Card + +Every benchmark has limitations. A benchmark caveat documents what the +benchmark does NOT prove. + +## Common Caveats + +| Caveat | Example | +|--------|---------| +| Charge-inflated | AMP-vs-decoy AUROC 0.7792 is partially charge-driven | +| Composition bias | Most features are composition-only, not order-aware | +| Small-n | Confidence intervals may be wide | +| Synthetic negatives | Decoys are not biologically validated | +| No wet-lab confirmation | All scores are computational predictions | + +## Rules + +- Every benchmark result must state its caveats. +- Caveats cannot be hidden in supplementary materials. +- A benchmark without caveats is incomplete. + +## Related + +- `docs/evidence/BENCHMARK_GOVERNANCE.md` — benchmark policies +- `docs/evidence/METRICS_CURRENT.md` — current results with caveats +- `docs/evidence/LIMITATIONS_OVERVIEW.md` — full limitations list diff --git a/docs/evidence/CONCEPT_CARD_MANIFEST.md b/docs/evidence/CONCEPT_CARD_MANIFEST.md new file mode 100644 index 00000000..662595e6 --- /dev/null +++ b/docs/evidence/CONCEPT_CARD_MANIFEST.md @@ -0,0 +1,25 @@ +# Manifest — Concept Card + +A manifest records what inputs, configs, and outputs a pipeline run used. + +## Required Fields + +| Field | Description | +|-------|-------------| +| `run_id` | UUID for the run | +| `pipeline_version` | Version string | +| `config_hash` | SHA-256 of config | +| `generated_at` | ISO 8601 timestamp | +| `inputs` | List of input file paths | +| `input_hashes` | SHA-256 per input file | +| `outputs` | List of output paths | + +## Purpose + +Manifests enable reproducibility audits. A reviewer can verify that the same +inputs and config produce the same outputs. + +## Related + +- `schemas/run_manifest.schema.json` — manifest schema +- `docs/engineering/RUN_MANIFEST_STANDARD.md` — full spec diff --git a/docs/evidence/CONCEPT_CARD_PROOF_LADDER.md b/docs/evidence/CONCEPT_CARD_PROOF_LADDER.md new file mode 100644 index 00000000..fefe4470 --- /dev/null +++ b/docs/evidence/CONCEPT_CARD_PROOF_LADDER.md @@ -0,0 +1,25 @@ +# Proof Ladder — Concept Card + +The proof ladder defines evidence levels for claims made about pipeline outputs. + +## Levels + +| Level | Name | Meaning | +|-------|------|---------| +| 0 | Syntax valid | Sequence uses canonical amino acids | +| 1 | Reproducible features | Features are deterministic | +| 2 | Baseline score | Heuristic passes basic filters | +| 3 | Multi-model consensus | Independent predictors agree | +| 4 | Leakage-resistant benchmark | Recovers hidden positives | +| 5 | Lab assay | Independent biological test | +| 6 | Independent replication | Second lab reproduces result | + +## Key Rule + +Computational scores (levels 0–4) are not biological proof. Only levels 5–6 +constitute biological evidence. + +## Related + +- `docs/evidence/PROOF_LADDER.md` — full specification +- `docs/evidence/LIMITATIONS_OVERVIEW.md` — current limitations diff --git a/docs/evidence/CONCEPT_CARD_SCHEMA.md b/docs/evidence/CONCEPT_CARD_SCHEMA.md new file mode 100644 index 00000000..360c293c --- /dev/null +++ b/docs/evidence/CONCEPT_CARD_SCHEMA.md @@ -0,0 +1,22 @@ +# Schema — Concept Card + +A schema is a machine-readable definition of an artifact's structure. + +## Schema Formats Used + +| Format | Use | Validator | +|--------|-----|-----------| +| JSON Schema (Draft 2020-12) | Evidence certs, lab results, manifests | `openamp-foundry validate` | +| Python dataclass | Simulation results, scored candidates | Runtime type checking | +| YAML | Config files, pass/fail criteria | Application-level loading | + +## Rules + +- Every persistent artifact should have a schema. +- Schema changes must be backward-compatible or versioned. +- Schemas live in `schemas/`. + +## Related + +- `schemas/` — schema directory +- `docs/engineering/SCHEMA_REGISTRY.md` — schema registry diff --git a/docs/evidence/RELEASE_STATUS_CONCEPT_CARD.md b/docs/evidence/RELEASE_STATUS_CONCEPT_CARD.md new file mode 100644 index 00000000..2d07beff --- /dev/null +++ b/docs/evidence/RELEASE_STATUS_CONCEPT_CARD.md @@ -0,0 +1,31 @@ +# Release Status Concept Card + +Defines what each release status means for downstream consumers. + +--- + +## Statuses + +| Status | Meaning | Can be cited? | Can be shared? | +|--------|---------|:-------------:|:--------------:| +| **Draft** | Work in progress, may change without notice | No | Internally only | +| **Internal** | Reviewed within the project team | With caveats | Within project | +| **Review-only** | Shared with qualified reviewers | In review context | With reviewers | +| **Public-safe** | Safe for public sharing | Yes, with limitations | Publicly | +| **Restricted** | Contains sensitive information | No | Authorized only | +| **Archived** | Superseded but preserved for reference | As historical | Yes, as-is | +| **Deprecated** | No longer maintained or recommended | Do not cite | Avoid | + +## Common Mistakes + +| Mistake | Correction | +|---------|------------| +| Citing a Draft as evidence | Use "Internal" or higher | +| Sharing Review-only publicly | Wait for "Public-safe" status | +| Treating Archived as current | Check for superseding document | +| Ignoring Restricted labels | Review access permissions | + +## Related + +- `docs/trust/RELEASE_CHECKLIST.md` — release process +- `docs/review/PUBLICATION_PACK.md` — publication preparation diff --git a/tests/test_concept_cards.py b/tests/test_concept_cards.py new file mode 100644 index 00000000..21fe53b9 --- /dev/null +++ b/tests/test_concept_cards.py @@ -0,0 +1,27 @@ +"""Tests for concept cards.""" +from pathlib import Path + +CARDS = [ + "CONCEPT_CARD_PROOF_LADDER.md", + "CONCEPT_CARD_ARTIFACT.md", + "CONCEPT_CARD_SCHEMA.md", + "CONCEPT_CARD_MANIFEST.md", + "CONCEPT_CARD_BENCHMARK_CAVEAT.md", +] + + +def test_all_exist(): + for card in CARDS: + assert Path(f"docs/evidence/{card}").exists(), f"Missing: {card}" + + +def test_each_has_related(): + for card in CARDS: + text = Path(f"docs/evidence/{card}").read_text() + assert "Related" in text, f"{card} missing Related section" + + +def test_none_overclaim(): + for card in CARDS: + text = Path(f"docs/evidence/{card}").read_text() + assert "proven" not in text.lower(), f"{card} contains 'proven'"