diff --git a/Makefile b/Makefile index edd45c38..6116e234 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -890,3 +890,8 @@ selection-rationale-check: .venv/bin/openamp-foundry selection-rationale-check \ --entry-json '{"selection_id":"SEL-2026-001","batch_id":"batch-2026-001","candidate_id":"OAMP-001","pipeline_version":"v0.7.9","selection_date":"2026-07-09","evidence_level":3,"baseline_comparison":"Scored above random baseline (AUROC 0.83) and charge-matched baseline (AUROC 0.71).","primary_criterion":"Top ensemble score with selectivity index > 2.","safety_flags_checked":["hemolysis_checked","toxicity_checked"],"reviewer":"maintainer-1"}' @echo "Selection rationale check complete." + +batch-priority-check: + .venv/bin/openamp-foundry batch-priority-check \ + --entry-json '{"priority_id":"PRI-2026-001","batch_id":"batch-2026-001","candidate_id":"OAMP-001","pipeline_version":"v0.8.0","priority_rank":1,"priority_score":0.87,"evidence_level":3,"synthesis_complexity":"low","novelty_tier":"high","primary_rationale":"Top ensemble score with selectivity > 2, low synthesis risk."}' + @echo "Batch priority check complete." diff --git a/docs/evidence/BATCH_PRIORITY_GUIDE.md b/docs/evidence/BATCH_PRIORITY_GUIDE.md new file mode 100644 index 00000000..d0a67ea7 --- /dev/null +++ b/docs/evidence/BATCH_PRIORITY_GUIDE.md @@ -0,0 +1,53 @@ +# Batch Experiment Priority Guide + +> **Scope:** Dry-lab only. No wet-lab or in-vivo claims. +> **Purpose:** Record the prioritization rationale for each candidate in a synthesis batch. +> When lab capacity is limited, external collaborators need a machine-validated priority +> ranking with explicit rationale, evidence level, and synthesis complexity. + +--- + +## What Is a Batch Priority Entry? + +A `BatchPriorityEntry` records, for a single candidate within a batch: + +1. Its priority rank among all candidates (1 = highest priority). +2. A 0–1 priority score summarizing the evidence and selection signals. +3. The evidence level (1–6 from `PROOF_LADDER.md`) supporting the selection. +4. Synthesis complexity estimate (low/medium/high). +5. Novelty tier relative to known AMPs (high/medium/low). +6. The primary rationale for the ranking. +7. Any disqualifying concerns that were considered and weighed. + +## Required Fields + +| Field | Description | +|---|---| +| `priority_id` | Unique ID, must start with `PRI-` | +| `batch_id` | The batch this priority entry belongs to | +| `candidate_id` | The specific candidate | +| `pipeline_version` | Pipeline version used for scoring | +| `priority_rank` | Integer ≥ 1 (1 = highest priority) | +| `priority_score` | Float 0.0–1.0 | +| `evidence_level` | 1–6, must match `PROOF_LADDER.md` | +| `synthesis_complexity` | `low`, `medium`, or `high` | +| `novelty_tier` | `high`, `medium`, or `low` | +| `primary_rationale` | Non-empty explanation of ranking | +| `dry_lab_only` | Must be True | + +## How to Validate + +```bash +make batch-priority-check +``` + +or: + +```bash +openamp-foundry batch-priority-check \ + --entry-json '{\"priority_id\": \"PRI-2026-001\", ...}' +``` + +--- + +*This guide is machine-validated. See `src/openamp_foundry/evidence/batch_priority.py`.* diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index bd14919c..082cbed8 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,9 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase K K1 — selection rationale schema — v0.7.9) +> **Last updated:** 2026-07-09 (Phase K K2 — batch experiment priority ranker — v0.8.0) +> **New in v0.8.0:** Batch experiment priority ranker (K2 — Loop 120) — docs/evidence/BATCH_PRIORITY_GUIDE.md with purpose, required field table (11 fields), how-to-validate instructions. src/openamp_foundry/evidence/batch_priority.py with BatchPriorityEntry dataclass (12 fields: priority_id, batch_id, candidate_id, pipeline_version, priority_rank, priority_score, evidence_level, synthesis_complexity, novelty_tier, primary_rationale, disqualifying_concerns, dry_lab_only), BatchPriorityResult dataclass (6 fields, dry_lab_only=True), VALID_SYNTHESIS_COMPLEXITIES (3: high, low, medium), VALID_NOVELTY_TIERS (3: high, low, medium), VALID_EVIDENCE_LEVELS (6: 1-6), validate_batch_priority() (11 checks: PRI- prefix, non-empty batch_id/candidate_id/pipeline_version, priority_rank>=1, priority_score 0.0-1.0, valid evidence_level, valid synthesis_complexity, valid novelty_tier, non-empty primary_rationale, dry_lab_only must be True; evidence_level<=2 warns, rank==1+high-complexity warns, priority_score<0.30 warns), validate_batch_priority_dict() (10 required fields guard). CLI (openamp-foundry batch-priority-check) with --entry-json, --format text|json. make batch-priority-check target. Phase K K2: synthesis wave ranking is now machine-validated. +> > **New in v0.7.9:** Candidate selection rationale schema (K1 — Loop 119) — docs/evidence/SELECTION_RATIONALE_GUIDE.md with purpose, required field table (11 fields), evidence level table (6 levels from PROOF_LADDER.md), validation workflow. src/openamp_foundry/evidence/selection_rationale.py with SelectionRationaleEntry dataclass (11 fields: selection_id, batch_id, candidate_id, pipeline_version, selection_date, evidence_level, baseline_comparison, primary_criterion, safety_flags_checked, reviewer, dry_lab_only), SelectionRationaleResult dataclass (5 fields, dry_lab_only=True), VALID_EVIDENCE_LEVELS (6: 1-6), MINIMUM_SAFETY_FLAGS (1), validate_selection_rationale() (11 checks: SEL- prefix, non-empty batch_id/candidate_id/pipeline_version, YYYY-MM-DD date, valid evidence_level 1-6, non-empty baseline_comparison/primary_criterion, safety_flags_checked list with >=1 entry, non-empty reviewer, dry_lab_only must be True; evidence_level<=2 warns with synthesis caution), validate_selection_rationale_dict() (10 required fields guard). CLI (openamp-foundry selection-rationale-check) with --entry-json, --format text|json. make selection-rationale-check target. Phase K K1: every selection decision now has a machine-validated evidence trail. > > **New in v0.7.8:** Annual safety and benchmark review checklist (J10 — Loop 118) — docs/governance/ANNUAL_REVIEW_CHECKLIST.md with 5-section structured checklist (safety_policy: 6 checks including dual-use safeguards, dry_lab_only enforcement, toxicity filters; benchmark_thresholds: 6 checks including threshold loosening guard, easy-baseline requirement, deprecation check; calibration_status: 4 checks including recalibration gate and rollback plan; governance_decisions: 4 checks including COI disclosures and rotation plan; data_governance: 3 checks including proprietary data flag). src/openamp_foundry/governance/annual_review.py with AnnualReviewEntry dataclass (10 fields: review_id, year, section, reviewer, finding_count, action_items_count, status, notes, completion_date, dry_lab_only), AnnualReviewResult dataclass (6 fields: review_id, year, section, passed, errors, warnings, dry_lab_only=True), VALID_REVIEW_SECTIONS (5: benchmark_thresholds, calibration_status, data_governance, governance_decisions, safety_policy), VALID_ENTRY_STATUSES (5: completed, deferred, in_progress, not_applicable, pending), validate_annual_review_entry() (9 checks: ANN- prefix, 4-digit year, valid section, non-empty reviewer, non-negative finding_count, non-negative action_items_count, valid status, completed requires YYYY-MM-DD completion_date, dry_lab_only must be True; completed+no-notes warns, deferred warns, findings+no-action-items warns), validate_annual_review_dict() (7 required fields guard). CLI (openamp-foundry annual-review-check) with --entry-json, --format text|json. make annual-review-check target. Long-term trust: annual review entries are now machine-validated. diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 906a824f..e7f0d380 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,21 @@ # Roadmap +## v0.8.0 — Loop 120: Phase K K2 — Batch Experiment Priority Ranker ✓ (2026-07-09) + +`docs/evidence/BATCH_PRIORITY_GUIDE.md` with field table and validation workflow +for batch synthesis wave priority entries. + +`src/openamp_foundry/evidence/batch_priority.py` with `BatchPriorityEntry` +dataclass (12 fields, dry_lab_only=True enforced), `BatchPriorityResult` +dataclass (6 fields), `VALID_SYNTHESIS_COMPLEXITIES` (3: high/low/medium), +`VALID_NOVELTY_TIERS` (3: high/low/medium), `VALID_EVIDENCE_LEVELS` (1–6), +`validate_batch_priority()` (11 checks, 3 warning conditions: low evidence, +top-rank+high-complexity, low score), `validate_batch_priority_dict()` +(10 required fields guard). +CLI: `openamp-foundry batch-priority-check`. `make batch-priority-check` target. +**v0.8.0 milestone** — synthesis wave ranking is now machine-validated with +explicit evidence level and complexity signals. + ## v0.7.9 — Loop 119: Phase K K1 — Selection Rationale Schema ✓ (2026-07-09) `docs/evidence/SELECTION_RATIONALE_GUIDE.md` documenting what selection rationale diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index e8ac1c87..d246e862 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -2851,3 +2851,37 @@ def _run_selection_rationale_check(args: argparse.Namespace) -> int: print(" Selection rationale validated. Decision is auditable.") return 0 if result.passed else 3 + + +def _run_batch_priority_check(args: argparse.Namespace) -> int: + """Validate a batch experiment priority entry.""" + import json as _json + from openamp_foundry.evidence.batch_priority import validate_batch_priority_dict + + try: + d = _json.loads(args.entry_json) + except _json.JSONDecodeError as exc: + print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"})) + return 2 + + if not isinstance(d, dict): + print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"})) + return 2 + + result = validate_batch_priority_dict(d) + output_format = getattr(args, "format", "text") + + if output_format == "json": + import dataclasses + print(_json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + print(f"Batch priority {result.priority_id} (candidate {result.candidate_id}, rank {result.priority_rank}): {status}") + for e in result.errors: + print(f" ERROR: {e}") + for w in result.warnings: + print(f" WARN: {w}") + if result.passed: + print(" Batch priority validated. Synthesis wave ranking is auditable.") + + return 0 if result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index d768ff31..2a5ec9de 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -15,7 +15,7 @@ from openamp_foundry.cli.commands.reports import _run_security_report_check from openamp_foundry.cli.commands.reports import _run_citation_check from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check -from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check +from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check import argparse import json @@ -1999,6 +1999,24 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Batch priority check (Phase K K2) ──────────────────────────── + bpc = sub.add_parser( + "batch-priority-check", + help=( + "Validate a batch experiment priority entry. Ranks candidates for " + "the next synthesis wave with evidence level and complexity. Dry-lab only." + ), + ) + bpc.add_argument( + "--entry-json", type=str, required=True, + help="JSON string of a BatchPriorityEntry dict (required).", + ) + bpc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", @@ -2307,6 +2325,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "selection-rationale-check": return _run_selection_rationale_check(args) + if args.command == "batch-priority-check": + return _run_batch_priority_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/evidence/batch_priority.py b/src/openamp_foundry/evidence/batch_priority.py new file mode 100644 index 00000000..5446e3b6 --- /dev/null +++ b/src/openamp_foundry/evidence/batch_priority.py @@ -0,0 +1,178 @@ +"""Batch experiment priority ranker for OpenAMP Foundry candidate synthesis planning.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List + +VALID_SYNTHESIS_COMPLEXITIES: set[str] = { + "high", + "low", + "medium", +} +VALID_NOVELTY_TIERS: set[str] = { + "high", + "low", + "medium", +} +VALID_EVIDENCE_LEVELS: set[int] = {1, 2, 3, 4, 5, 6} + + +@dataclass +class BatchPriorityEntry: + """A priority ranking entry for one candidate in a synthesis batch.""" + + priority_id: str + batch_id: str + candidate_id: str + pipeline_version: str + priority_rank: int + priority_score: float + evidence_level: int + synthesis_complexity: str + novelty_tier: str + primary_rationale: str + disqualifying_concerns: List[str] = field(default_factory=list) + dry_lab_only: bool = True + + +@dataclass +class BatchPriorityResult: + """Result of validating a BatchPriorityEntry.""" + + priority_id: str + candidate_id: str + priority_rank: int + passed: bool + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + dry_lab_only: bool = True + + +def validate_batch_priority(entry: BatchPriorityEntry) -> BatchPriorityResult: + """Validate a BatchPriorityEntry against policy rules.""" + errors: list[str] = [] + warnings: list[str] = [] + + if not entry.priority_id or not str(entry.priority_id).strip(): + errors.append("priority_id must be non-empty") + elif not entry.priority_id.startswith("PRI-"): + errors.append( + f"priority_id must start with 'PRI-', got: {entry.priority_id!r}" + ) + + if not entry.batch_id or not str(entry.batch_id).strip(): + errors.append("batch_id must be non-empty") + + if not entry.candidate_id or not str(entry.candidate_id).strip(): + errors.append("candidate_id must be non-empty") + + if not entry.pipeline_version or not str(entry.pipeline_version).strip(): + errors.append("pipeline_version must be non-empty") + + if not isinstance(entry.priority_rank, int) or entry.priority_rank < 1: + errors.append( + f"priority_rank must be an integer >= 1, got: {entry.priority_rank!r}" + ) + + if not isinstance(entry.priority_score, (int, float)) or not (0.0 <= float(entry.priority_score) <= 1.0): + errors.append( + f"priority_score must be a float between 0.0 and 1.0, got: {entry.priority_score!r}" + ) + + if not isinstance(entry.evidence_level, int) or entry.evidence_level not in VALID_EVIDENCE_LEVELS: + errors.append( + f"evidence_level must be one of {sorted(VALID_EVIDENCE_LEVELS)}, " + f"got: {entry.evidence_level!r}" + ) + + if entry.synthesis_complexity not in VALID_SYNTHESIS_COMPLEXITIES: + errors.append( + f"synthesis_complexity must be one of {sorted(VALID_SYNTHESIS_COMPLEXITIES)}, " + f"got: {entry.synthesis_complexity!r}" + ) + + if entry.novelty_tier not in VALID_NOVELTY_TIERS: + errors.append( + f"novelty_tier must be one of {sorted(VALID_NOVELTY_TIERS)}, " + f"got: {entry.novelty_tier!r}" + ) + + if not entry.primary_rationale or not str(entry.primary_rationale).strip(): + errors.append("primary_rationale must be non-empty") + + if not entry.dry_lab_only: + errors.append("dry_lab_only must be True for batch priority entries") + + # Warnings + if isinstance(entry.evidence_level, int) and entry.evidence_level <= 2: + warnings.append( + f"evidence_level is {entry.evidence_level} (low): " + "consider additional evidence before committing to synthesis" + ) + + if ( + isinstance(entry.priority_rank, int) + and entry.priority_rank == 1 + and entry.synthesis_complexity == "high" + ): + warnings.append( + "top-ranked candidate has high synthesis complexity: " + "confirm synthesis feasibility before ordering" + ) + + if isinstance(entry.priority_score, (int, float)) and float(entry.priority_score) < 0.3: + warnings.append( + f"priority_score is {entry.priority_score:.2f} (below 0.30): " + "consider whether this candidate meets the synthesis threshold" + ) + + return BatchPriorityResult( + priority_id=entry.priority_id, + candidate_id=entry.candidate_id, + priority_rank=entry.priority_rank, + passed=len(errors) == 0, + errors=errors, + warnings=warnings, + dry_lab_only=True, + ) + + +def validate_batch_priority_dict(d: dict) -> BatchPriorityResult: + """Validate a dict representation of a BatchPriorityEntry.""" + required = [ + "batch_id", + "candidate_id", + "evidence_level", + "novelty_tier", + "pipeline_version", + "primary_rationale", + "priority_id", + "priority_rank", + "priority_score", + "synthesis_complexity", + ] + missing = [f for f in required if f not in d] + if missing: + return BatchPriorityResult( + priority_id=d.get("priority_id", ""), + candidate_id=d.get("candidate_id", ""), + priority_rank=d.get("priority_rank", 0), + passed=False, + errors=[f"Missing required fields: {missing}"], + dry_lab_only=True, + ) + entry = BatchPriorityEntry( + priority_id=d["priority_id"], + batch_id=d["batch_id"], + candidate_id=d["candidate_id"], + pipeline_version=d["pipeline_version"], + priority_rank=d["priority_rank"], + priority_score=d["priority_score"], + evidence_level=d["evidence_level"], + synthesis_complexity=d["synthesis_complexity"], + novelty_tier=d["novelty_tier"], + primary_rationale=d["primary_rationale"], + disqualifying_concerns=d.get("disqualifying_concerns", []), + dry_lab_only=d.get("dry_lab_only", True), + ) + return validate_batch_priority(entry) diff --git a/tests/evidence/test_batch_priority.py b/tests/evidence/test_batch_priority.py new file mode 100644 index 00000000..8cfb988b --- /dev/null +++ b/tests/evidence/test_batch_priority.py @@ -0,0 +1,247 @@ +"""Tests for batch experiment priority ranker validation.""" +import pytest +from openamp_foundry.evidence.batch_priority import ( + VALID_EVIDENCE_LEVELS, + VALID_NOVELTY_TIERS, + VALID_SYNTHESIS_COMPLEXITIES, + BatchPriorityEntry, + BatchPriorityResult, + validate_batch_priority, + validate_batch_priority_dict, +) + + +def _valid_entry(**kwargs) -> BatchPriorityEntry: + defaults = dict( + priority_id="PRI-2026-001", + batch_id="batch-2026-001", + candidate_id="OAMP-001", + pipeline_version="v0.8.0", + priority_rank=1, + priority_score=0.87, + evidence_level=3, + synthesis_complexity="low", + novelty_tier="high", + primary_rationale="Top ensemble score with selectivity > 2, low synthesis risk.", + disqualifying_concerns=[], + dry_lab_only=True, + ) + defaults.update(kwargs) + return BatchPriorityEntry(**defaults) + + +class TestValidBatchPriority: + def test_valid_entry_passes(self): + result = validate_batch_priority(_valid_entry()) + assert result.passed is True + assert result.errors == [] + + def test_result_has_correct_priority_id(self): + result = validate_batch_priority(_valid_entry()) + assert result.priority_id == "PRI-2026-001" + + def test_result_has_correct_candidate_id(self): + result = validate_batch_priority(_valid_entry()) + assert result.candidate_id == "OAMP-001" + + def test_result_has_correct_priority_rank(self): + result = validate_batch_priority(_valid_entry()) + assert result.priority_rank == 1 + + def test_all_evidence_levels_accepted(self): + for level in VALID_EVIDENCE_LEVELS: + result = validate_batch_priority(_valid_entry(evidence_level=level)) + assert result.passed is True, f"Expected level {level} to pass" + + def test_all_synthesis_complexities_accepted(self): + for sc in VALID_SYNTHESIS_COMPLEXITIES: + result = validate_batch_priority(_valid_entry(synthesis_complexity=sc)) + assert result.passed is True, f"Expected complexity {sc!r} to pass" + + def test_all_novelty_tiers_accepted(self): + for nt in VALID_NOVELTY_TIERS: + result = validate_batch_priority(_valid_entry(novelty_tier=nt)) + assert result.passed is True, f"Expected novelty tier {nt!r} to pass" + + def test_dry_lab_only_is_true_in_result(self): + result = validate_batch_priority(_valid_entry()) + assert result.dry_lab_only is True + + def test_priority_rank_10_passes(self): + result = validate_batch_priority(_valid_entry(priority_rank=10)) + assert result.passed is True + + def test_priority_score_zero_passes(self): + result = validate_batch_priority(_valid_entry(priority_score=0.0)) + assert result.passed is True + + def test_priority_score_one_passes(self): + result = validate_batch_priority(_valid_entry(priority_score=1.0)) + assert result.passed is True + + def test_with_disqualifying_concerns_passes(self): + result = validate_batch_priority( + _valid_entry(disqualifying_concerns=["hemolysis risk at high dose, considered acceptable"]) + ) + assert result.passed is True + + +class TestInvalidBatchPriority: + def test_empty_priority_id_fails(self): + result = validate_batch_priority(_valid_entry(priority_id="")) + assert result.passed is False + assert any("priority_id" in e for e in result.errors) + + def test_priority_id_without_pri_prefix_fails(self): + result = validate_batch_priority(_valid_entry(priority_id="CAND-2026-001")) + assert result.passed is False + assert any("PRI-" in e for e in result.errors) + + def test_empty_batch_id_fails(self): + result = validate_batch_priority(_valid_entry(batch_id="")) + assert result.passed is False + assert any("batch_id" in e for e in result.errors) + + def test_empty_candidate_id_fails(self): + result = validate_batch_priority(_valid_entry(candidate_id="")) + assert result.passed is False + assert any("candidate_id" in e for e in result.errors) + + def test_empty_pipeline_version_fails(self): + result = validate_batch_priority(_valid_entry(pipeline_version="")) + assert result.passed is False + assert any("pipeline_version" in e for e in result.errors) + + def test_priority_rank_zero_fails(self): + result = validate_batch_priority(_valid_entry(priority_rank=0)) + assert result.passed is False + assert any("priority_rank" in e for e in result.errors) + + def test_priority_rank_negative_fails(self): + result = validate_batch_priority(_valid_entry(priority_rank=-1)) + assert result.passed is False + assert any("priority_rank" in e for e in result.errors) + + def test_priority_score_negative_fails(self): + result = validate_batch_priority(_valid_entry(priority_score=-0.1)) + assert result.passed is False + assert any("priority_score" in e for e in result.errors) + + def test_priority_score_above_one_fails(self): + result = validate_batch_priority(_valid_entry(priority_score=1.1)) + assert result.passed is False + assert any("priority_score" in e for e in result.errors) + + def test_invalid_evidence_level_fails(self): + result = validate_batch_priority(_valid_entry(evidence_level=7)) + assert result.passed is False + assert any("evidence_level" in e for e in result.errors) + + def test_invalid_synthesis_complexity_fails(self): + result = validate_batch_priority(_valid_entry(synthesis_complexity="very_high")) + assert result.passed is False + assert any("synthesis_complexity" in e for e in result.errors) + + def test_invalid_novelty_tier_fails(self): + result = validate_batch_priority(_valid_entry(novelty_tier="very_high")) + assert result.passed is False + assert any("novelty_tier" in e for e in result.errors) + + def test_empty_primary_rationale_fails(self): + result = validate_batch_priority(_valid_entry(primary_rationale="")) + assert result.passed is False + assert any("primary_rationale" in e for e in result.errors) + + def test_dry_lab_only_false_fails(self): + result = validate_batch_priority(_valid_entry(dry_lab_only=False)) + assert result.passed is False + assert any("dry_lab_only" in e for e in result.errors) + + +class TestBatchPriorityWarnings: + def test_low_evidence_level_warns(self): + result = validate_batch_priority(_valid_entry(evidence_level=1)) + assert result.passed is True + assert any("evidence_level" in w for w in result.warnings) + + def test_evidence_level_2_warns(self): + result = validate_batch_priority(_valid_entry(evidence_level=2)) + assert result.passed is True + assert any("evidence_level" in w for w in result.warnings) + + def test_evidence_level_3_no_warning(self): + result = validate_batch_priority(_valid_entry(evidence_level=3)) + assert result.passed is True + assert not any("evidence_level" in w for w in result.warnings) + + def test_top_rank_high_complexity_warns(self): + result = validate_batch_priority( + _valid_entry(priority_rank=1, synthesis_complexity="high") + ) + assert result.passed is True + assert any("synthesis" in w for w in result.warnings) + + def test_rank2_high_complexity_no_warn(self): + result = validate_batch_priority( + _valid_entry(priority_rank=2, synthesis_complexity="high") + ) + assert result.passed is True + assert not any("synthesis" in w for w in result.warnings) + + def test_low_priority_score_warns(self): + result = validate_batch_priority(_valid_entry(priority_score=0.2)) + assert result.passed is True + assert any("priority_score" in w for w in result.warnings) + + def test_score_0_3_no_warn(self): + result = validate_batch_priority(_valid_entry(priority_score=0.3)) + assert result.passed is True + assert not any("priority_score" in w for w in result.warnings) + + def test_no_warnings_for_clean_entry(self): + result = validate_batch_priority(_valid_entry()) + assert result.warnings == [] + + +class TestValidateBatchPriorityDict: + def test_valid_dict_passes(self): + d = { + "priority_id": "PRI-2026-002", + "batch_id": "batch-2026-001", + "candidate_id": "OAMP-002", + "pipeline_version": "v0.8.0", + "priority_rank": 2, + "priority_score": 0.75, + "evidence_level": 4, + "synthesis_complexity": "medium", + "novelty_tier": "medium", + "primary_rationale": "Strong calibrated score, medium synthesis complexity.", + "dry_lab_only": True, + } + result = validate_batch_priority_dict(d) + assert result.passed is True + + def test_missing_fields_fails(self): + result = validate_batch_priority_dict({"priority_id": "PRI-2026-003"}) + assert result.passed is False + assert any("Missing required fields" in e for e in result.errors) + + def test_empty_dict_fails(self): + result = validate_batch_priority_dict({}) + assert result.passed is False + + def test_dict_defaults_dry_lab_only_true(self): + d = { + "priority_id": "PRI-2026-004", + "batch_id": "batch-2026-002", + "candidate_id": "OAMP-003", + "pipeline_version": "v0.8.0", + "priority_rank": 3, + "priority_score": 0.65, + "evidence_level": 3, + "synthesis_complexity": "low", + "novelty_tier": "high", + "primary_rationale": "Novel scaffold with acceptable risk.", + } + result = validate_batch_priority_dict(d) + assert result.dry_lab_only is True diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index ff40fe27..50b69ed4 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3711 +BASELINE = 3749 TOLERANCE = 0.05 # allow 5% fluctuation