From 67504f3dcdbafd7b776d5b31acc00a6a2ac91097 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Wed, 8 Jul 2026 22:22:19 +0700 Subject: [PATCH] Add 10 calibration and selection files Closes #285, Closes #286, Closes #287, Closes #288, Closes #289, Closes #290, Closes #291, Closes #292, Closes #293, Closes #295 --- docs/evidence/GUIDE_ACTIVE_LEARNING_TRACE.md | 30 ++++++++++++++++++ .../GUIDE_CALIBRATION_PROPOSAL_PROVENANCE.md | 20 ++++++++++++ docs/evidence/GUIDE_CALIBRATION_QUARANTINE.md | 29 +++++++++++++++++ docs/evidence/GUIDE_COHORT_SUFFICIENCY.md | 21 +++++++++++++ docs/evidence/GUIDE_SELECTION_AUDIT.md | 31 +++++++++++++++++++ docs/evidence/TAXONOMY_EXCLUSION_REASONS.md | 21 +++++++++++++ .../evidence/TEMPLATE_CALIBRATION_ROLLBACK.md | 30 ++++++++++++++++++ tests/test_benchmark_slice_coverage.py | 24 ++++++++++++++ tests/test_public_redaction.py | 17 ++++++++++ tests/test_synthetic_quarantine.py | 20 ++++++++++++ 10 files changed, 243 insertions(+) create mode 100644 docs/evidence/GUIDE_ACTIVE_LEARNING_TRACE.md create mode 100644 docs/evidence/GUIDE_CALIBRATION_PROPOSAL_PROVENANCE.md create mode 100644 docs/evidence/GUIDE_CALIBRATION_QUARANTINE.md create mode 100644 docs/evidence/GUIDE_COHORT_SUFFICIENCY.md create mode 100644 docs/evidence/GUIDE_SELECTION_AUDIT.md create mode 100644 docs/evidence/TAXONOMY_EXCLUSION_REASONS.md create mode 100644 docs/evidence/TEMPLATE_CALIBRATION_ROLLBACK.md create mode 100644 tests/test_benchmark_slice_coverage.py create mode 100644 tests/test_public_redaction.py create mode 100644 tests/test_synthetic_quarantine.py diff --git a/docs/evidence/GUIDE_ACTIVE_LEARNING_TRACE.md b/docs/evidence/GUIDE_ACTIVE_LEARNING_TRACE.md new file mode 100644 index 00000000..0221369d --- /dev/null +++ b/docs/evidence/GUIDE_ACTIVE_LEARNING_TRACE.md @@ -0,0 +1,30 @@ +# Active-Learning Decision Trace Artifact + +Records the decision process for active-learning candidate selection. + +## Fields +| Field | Description | +|-------|-------------| +| round | Active-learning round number | +| candidates_considered | How many candidates were evaluated | +| acquisition_function | What strategy was used (uncertainty, diversity, etc.) | +| selected_candidates | Which candidates were selected | +| selection_scores | Scores used for selection | +| batch_constraints | Batch size, budget, diversity requirements | + +## Example +```json +{ + "round": 2, + "candidates_considered": 50, + "acquisition_function": "uncertainty_sampling", + "selected_candidates": ["C001", "C002", "C003"], + "selection_scores": {"disagreement": 0.45, "ensemble": 0.72}, + "batch_constraints": {"max_batch_size": 12, "min_diversity": 0.75} +} +``` + +## Rules +- Each active-learning round should produce a decision trace. +- The trace should include enough information to reproduce the selection. +- Selection criteria should be documented in the trace. diff --git a/docs/evidence/GUIDE_CALIBRATION_PROPOSAL_PROVENANCE.md b/docs/evidence/GUIDE_CALIBRATION_PROPOSAL_PROVENANCE.md new file mode 100644 index 00000000..c2587c47 --- /dev/null +++ b/docs/evidence/GUIDE_CALIBRATION_PROPOSAL_PROVENANCE.md @@ -0,0 +1,20 @@ +# Calibration Proposal Provenance Fields + +Required provenance fields for calibration proposals. + +## Required Fields +| Field | Description | Example | +|-------|-------------|---------| +| proposal_id | Unique identifier | CP-2026-07-001 | +| created_at | ISO 8601 timestamp | 2026-07-08T12:00:00Z | +| created_by | Human or agent name | Maintainer name | +| intake_report | Link to intake report | outputs/intake_report.json | +| gate_verdict | Link to gate verdict | outputs/gate_verdict.json | +| current_weights | Weights before change | {"activity": 0.40} | +| proposed_weights | Weights after change | {"activity": 0.42} | +| rationale | Why the change is needed | "Improve selectivity detection" | + +## Rules +- All fields are required for a valid proposal. +- Proposals without complete provenance must not be applied. +- Provenance fields should be included in the weight proposal JSON. diff --git a/docs/evidence/GUIDE_CALIBRATION_QUARANTINE.md b/docs/evidence/GUIDE_CALIBRATION_QUARANTINE.md new file mode 100644 index 00000000..89ef3e64 --- /dev/null +++ b/docs/evidence/GUIDE_CALIBRATION_QUARANTINE.md @@ -0,0 +1,29 @@ +# Calibration Input Quarantine Report + +Report on calibration inputs that were quarantined. + +## Quarantine Reasons +| Reason | Description | Action | +|--------|-------------|--------| +| Low quality | Result quality flag is low or invalid | Exclude from calibration | +| Control failure | Controls did not pass | Regenerate or exclude | +| Missing fields | Required fields absent | Request resubmission | +| Orphan result | No matching prediction | Investigate mismatch | + +## Report Format +```json +{ + "total_results": 20, + "quarantined": 2, + "reasons": [ + {"candidate_id": "C001", "reason": "control_failure"}, + {"candidate_id": "C002", "reason": "low_quality"} + ], + "passed_gate": false +} +``` + +## Rules +- Quarantined results must not be used for calibration. +- Quarantine reasons should be documented per result. +- If > 50% of results are quarantined, investigate the batch. diff --git a/docs/evidence/GUIDE_COHORT_SUFFICIENCY.md b/docs/evidence/GUIDE_COHORT_SUFFICIENCY.md new file mode 100644 index 00000000..42eb1285 --- /dev/null +++ b/docs/evidence/GUIDE_COHORT_SUFFICIENCY.md @@ -0,0 +1,21 @@ +# Calibration Cohort Sufficiency Explainer + +How cohort size affects calibration reliability. + +## Minimum Cohort +The recalibration gate requires a minimum cohort size of 5. +This prevents calibration from being influenced by individual outliers. + +## Why Size Matters +| Cohort Size | Reliability | Risk | +|:-----------:|:-----------:|:----:| +| 1-4 | Low | Single result can shift weights | +| 5-9 | Moderate | Some protection from outliers | +| 10-19 | Good | Reasonable statistical power | +| 20+ | High | Robust to individual outliers | + +## Rules +- Cohorts below minimum size are rejected by the gate. +- Results from small cohorts are informational only. +- Cohort size should be reported alongside calibration results. +- If the cohort is small, the calibration report should note the uncertainty. diff --git a/docs/evidence/GUIDE_SELECTION_AUDIT.md b/docs/evidence/GUIDE_SELECTION_AUDIT.md new file mode 100644 index 00000000..6b3d79ce --- /dev/null +++ b/docs/evidence/GUIDE_SELECTION_AUDIT.md @@ -0,0 +1,31 @@ +# Selection Audit Summary Report + +Summary of candidate selection decisions for audit. + +## Report Sections +- Total candidates considered +- Candidates passing each gate +- Candidates selected (final) +- Candidates excluded with reasons +- Diversity metrics of selected set + +## Example +```json +{ + "total_candidates": 100, + "passed_validity": 95, + "passed_novelty": 80, + "passed_safety": 65, + "passed_synthesis": 60, + "selected": 24, + "excluded": [ + {"candidate_id": "C001", "reason": "low_novelty"}, + {"candidate_id": "C002", "reason": "safety_risk"} + ] +} +``` + +## Rules +- The audit report must include exclusion reasons. +- Exclusion reasons should follow the exclusion reason taxonomy. +- The report should be generated when the final panel is selected. diff --git a/docs/evidence/TAXONOMY_EXCLUSION_REASONS.md b/docs/evidence/TAXONOMY_EXCLUSION_REASONS.md new file mode 100644 index 00000000..c72066ad --- /dev/null +++ b/docs/evidence/TAXONOMY_EXCLUSION_REASONS.md @@ -0,0 +1,21 @@ +# Exclusion Reason Taxonomy for Selection Workflows + +Standard exclusion reasons for candidate selection. + +| Reason | Description | Gate | +|--------|-------------|:----:| +| invalid_sequence | Non-canonical amino acids | validity | +| too_short | Below minimum length | validity | +| too_long | Above maximum length | validity | +| low_novelty | Near-duplicate of known AMP | novelty | +| safety_risk | Above safety threshold | safety | +| synthesis_risk | Above synthesis difficulty | synthesis | +| low_activity | Below activity threshold | activity | +| low_diversity | Too similar to selected candidate | diversity | +| budget_limit | Not enough budget for all candidates | selection | + +## Rules +- Every excluded candidate must have an exclusion reason. +- Exclusion reasons are recorded in the selection audit report. +- Multiple exclusion reasons can apply to a single candidate. +- If no reason applies, use `other` and document the reason. diff --git a/docs/evidence/TEMPLATE_CALIBRATION_ROLLBACK.md b/docs/evidence/TEMPLATE_CALIBRATION_ROLLBACK.md new file mode 100644 index 00000000..6bf43e66 --- /dev/null +++ b/docs/evidence/TEMPLATE_CALIBRATION_ROLLBACK.md @@ -0,0 +1,30 @@ +# Calibration Rollback Artifact Template + +Template for documenting calibration rollbacks. + +```markdown +# Calibration Rollback Record + +## Date +YYYY-MM-DD + +## Reason +Why the rollback was necessary. + +## Previous State +- Weights before rollback: [weights] +- Gate verdict: [passed/failed] + +## Current State +- Weights after rollback: [weights] +- Benchmarks affected: [list] + +## Verification +- [ ] `make bench-gate` passes +- [ ] `make full-reproducibility-report` succeeds +- [ ] Rollback logged in decision log + +## Related +- Link to decision log entry. +- Link to original calibration proposal. +``` diff --git a/tests/test_benchmark_slice_coverage.py b/tests/test_benchmark_slice_coverage.py new file mode 100644 index 00000000..47a9d6b7 --- /dev/null +++ b/tests/test_benchmark_slice_coverage.py @@ -0,0 +1,24 @@ +"""Test benchmark slice coverage.""" +from pathlib import Path + + +def test_per_family_benchmark_exists(): + """Per-family benchmark should have output.""" + result = Path("outputs/benchmark_per_family.json") + if not result.exists(): + return # May not be generated yet + import json + data = json.loads(result.read_text()) + assert len(data) > 0 + + +def test_metrics_current_has_per_family(): + text = Path("docs/evidence/METRICS_CURRENT.md").read_text() + assert "Per-Family" in text or "per-family" in text + + +def test_benchmark_slice_defined(): + text = Path("docs/evidence/GUIDE_BENCHMARK_SLICE.md").read_text() if Path( + "docs/evidence/GUIDE_BENCHMARK_SLICE.md").exists() else "" + if text: + assert "Slice" in text diff --git a/tests/test_public_redaction.py b/tests/test_public_redaction.py new file mode 100644 index 00000000..72d52387 --- /dev/null +++ b/tests/test_public_redaction.py @@ -0,0 +1,17 @@ +"""Test public artifact redaction policy.""" +import json +from pathlib import Path + + +def test_redaction_policy_exists(): + assert Path("docs/evidence/GUIDE_REDACTION_POLICY.md").exists() + + +def test_synthetic_data_policy_exists(): + assert Path("docs/evidence/SYNTHETIC_DATA_POLICY.md").exists() + + +def test_lab_result_schema_has_disclaimer(): + schema = json.loads(Path("schemas/lab_result.schema.json").read_text()) + props = schema.get("properties", {}) + assert "disclaimer" in props, "Lab result schema should have disclaimer field" diff --git a/tests/test_synthetic_quarantine.py b/tests/test_synthetic_quarantine.py new file mode 100644 index 00000000..ea96baa6 --- /dev/null +++ b/tests/test_synthetic_quarantine.py @@ -0,0 +1,20 @@ +"""Test synthetic result quarantine rules.""" +from pathlib import Path + + +def test_synthetic_lab_results_labeled(): + """Synthetic lab results should be clearly labeled.""" + lab_dir = Path("examples/lab_results") + if not lab_dir.exists(): + return + for f in lab_dir.glob("*.json"): + import json + data = json.loads(f.read_text()) + disclaimer = data.get("disclaimer", "") + assert "SYNTHETIC" in disclaimer or "synthetic" in disclaimer.lower(), \ + f"Missing SYNTHETIC label in {f.name}" + + +def test_synthetic_data_policy_mentions_quarantine(): + text = Path("docs/evidence/SYNTHETIC_DATA_POLICY.md").read_text() + assert "quarantine" in text.lower() or "label" in text.lower()