diff --git a/Makefile b/Makefile index 1882851c..34b8cdfe 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -925,3 +925,8 @@ candidate-summary-card-check: .venv/bin/openamp-foundry candidate-summary-card-check \ --entry-json '{"card_id":"CRD-001","candidate_id":"AMP-001","batch_id":"BATCH-001","pipeline_version":"0.8.6","sequence":"KWKLFKKIEK","sequence_length":10,"evidence_level":4,"predicted_activity":"high_activity","safety_flags":[],"selection_rationale_id":"SEL-001","reviewer":"alice","dry_lab_only":true}' \ --format text + +multi-candidate-comparison-check: + .venv/bin/openamp-foundry multi-candidate-comparison-check \ + --entry-json '{"comparison_id":"CMP-001","batch_id":"BATCH-001","pipeline_version":"0.8.7","comparison_date":"2026-07-09","candidate_ids":["AMP-001","AMP-002","AMP-003"],"comparison_criteria":["predicted_mic","hemolysis_fraction","selectivity_index"],"top_candidate_id":"AMP-001","top_candidate_rationale":"Highest selectivity index with lowest hemolysis fraction.","evidence_level":4,"reviewer":"alice","dry_lab_only":true}' \ + --format text diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index c04e3678..50244931 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase L L3 — candidate summary card schema — v0.8.6) +> **Last updated:** 2026-07-09 (Phase L L4 — multi-candidate comparison schema — v0.8.7) +> **New in v0.8.7:** Multi-candidate comparison schema (L4 — Loop 126b) — docs/evidence/MULTI_CANDIDATE_COMPARISON_GUIDE.md with purpose, required field table (11 fields), minimum requirements (2 candidates, 2 criteria), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/multi_candidate_comparison.py with MultiCandidateComparisonEntry dataclass (11 fields: comparison_id, batch_id, pipeline_version, comparison_date, candidate_ids, comparison_criteria, top_candidate_id, top_candidate_rationale, evidence_level, reviewer, dry_lab_only), MultiCandidateComparisonResult dataclass (5 fields: comparison_id, batch_id, candidate_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_CANDIDATES (2), MINIMUM_CRITERIA (2), RECOMMENDED_CRITERIA (3), MAX_RATIONALE_LENGTH (500), LARGE_CANDIDATE_SET_THRESHOLD (10), VALID_EVIDENCE_LEVELS (6: 1-6), validate_multi_candidate_comparison() (11 checks: CMP- prefix, non-empty batch_id/pipeline_version/reviewer, YYYY-MM-DD comparison_date, candidate_ids>=2, comparison_criteria>=2, top_candidate_id in candidate_ids, non-empty rationale<=500 chars, valid evidence_level, dry_lab_only must be True; evidence_level<=2 warns, candidate_count>10 warns, criteria_count<3 warns), validate_multi_candidate_comparison_dict() (10 required fields guard). CLI (openamp-foundry multi-candidate-comparison-check) with --entry-json, --format text|json. make multi-candidate-comparison-check target. Phase L L4: side-by-side candidate comparisons are now machine-validated for publication-ready supplementary tables. > **New in v0.8.6:** Candidate summary card schema (L3 — Loop 126) — docs/evidence/CANDIDATE_SUMMARY_CARD_GUIDE.md with purpose, required field table (12 fields), valid activity labels (5), valid amino acid set (20 standard), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/candidate_summary_card.py with CandidateSummaryCardEntry dataclass (12 fields: card_id, candidate_id, batch_id, pipeline_version, sequence, sequence_length, evidence_level, predicted_activity, safety_flags, selection_rationale_id, reviewer, dry_lab_only), CandidateSummaryCardResult dataclass (5 fields: card_id, candidate_id, sequence_length, passed, errors, warnings, dry_lab_only=True), VALID_ACTIVITY_LABELS (5: high_activity, inactive, low_activity, moderate_activity, uncertain), VALID_AMINO_ACIDS (20 standard one-letter codes), LONG_PEPTIDE_THRESHOLD (50), VALID_EVIDENCE_LEVELS (6: 1-6), validate_candidate_summary_card() (11 checks: CRD- prefix, non-empty candidate_id/batch_id/pipeline_version/reviewer, non-empty sequence with valid amino acids, sequence_length==len(sequence), valid evidence_level, valid predicted_activity, SEL- prefix on selection_rationale_id, dry_lab_only must be True; evidence_level<=2 warns, safety_flags non-empty warns, uncertain activity warns, length>50 warns), validate_candidate_summary_card_dict() (11 required fields guard). CLI (openamp-foundry candidate-summary-card-check) with --entry-json, --format text|json. make candidate-summary-card-check target. Phase L L3: every candidate now has a machine-validated publication-ready summary card. > **New in v0.8.5:** Reproducibility manifest schema (L2 — Loop 125) — docs/evidence/REPRODUCIBILITY_MANIFEST_GUIDE.md with purpose, required field table (11 fields), package checksums format, data checksums format, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/reproducibility_manifest.py with ReproducibilityManifestEntry dataclass (11 fields: manifest_id, batch_id, pipeline_version, run_date, python_version, package_checksums, data_checksums, random_seeds, hardware_summary, reviewer, dry_lab_only), ReproducibilityManifestResult dataclass (6 fields: manifest_id, batch_id, package_count, data_file_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_PACKAGES (3), MINIMUM_DATA_FILES (1), RECOMMENDED_PACKAGES (5), validate_reproducibility_manifest() (10 checks: RPM- prefix, non-empty batch_id/pipeline_version/python_version/hardware_summary/reviewer, YYYY-MM-DD run_date, package_checksums>=3, data_checksums>=1, dry_lab_only must be True; empty random_seeds warns, package_count<5 warns, hardware_summary contains 'unknown' warns), validate_reproducibility_manifest_dict() (10 required fields guard). CLI (openamp-foundry reproducibility-manifest-check) with --entry-json, --format text|json. make reproducibility-manifest-check target. Phase L L2: every pipeline run now has a machine-validated reproducibility record. > **New in v0.8.4:** Preprint evidence bundle schema (L1 — Loop 124) — docs/evidence/PREPRINT_BUNDLE_GUIDE.md with purpose, required field table (11 fields), minimum artifact count (3), evidence level guide, warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/preprint_bundle.py with PreprintBundleEntry dataclass (11 fields: bundle_id, batch_id, pipeline_version, submission_date, title, artifact_ids, evidence_level, preprint_doi, contact_email, release_approved, dry_lab_only), PreprintBundleResult dataclass (5 fields: bundle_id, batch_id, artifact_count, passed, errors, warnings, dry_lab_only=True), MINIMUM_ARTIFACTS (3), RECOMMENDED_ARTIFACT_COUNT (5), MAX_TITLE_LENGTH (300), VALID_EVIDENCE_LEVELS (6: 1-6), validate_preprint_bundle() (10 checks: BND- prefix, non-empty batch_id/pipeline_version/contact_email, YYYY-MM-DD submission_date, non-empty title<=300 chars, artifact_ids>=3, valid evidence_level 1-6, release_approved must be True, dry_lab_only must be True; evidence_level<=2 warns, empty preprint_doi warns, artifact_count<5 warns), validate_preprint_bundle_dict() (10 required fields guard). CLI (openamp-foundry preprint-bundle-check) with --entry-json, --format text|json. make preprint-bundle-check target. Phase L L1: scientific preprints now have a machine-validated evidence bundle structure. diff --git a/docs/evidence/MULTI_CANDIDATE_COMPARISON_GUIDE.md b/docs/evidence/MULTI_CANDIDATE_COMPARISON_GUIDE.md new file mode 100644 index 00000000..850cea13 --- /dev/null +++ b/docs/evidence/MULTI_CANDIDATE_COMPARISON_GUIDE.md @@ -0,0 +1,59 @@ +# Multi-Candidate Comparison Guide + +A *multi-candidate comparison* is a structured record that compares the key predicted +properties of two or more candidates from the same batch side-by-side. This enables +publication-ready supplementary tables and supports consistent external review. + +## Purpose + +When nominating multiple candidates from a single batch, reviewers need to: +- see how candidates differ across predicted properties, +- confirm that comparison criteria are consistent across all candidates, +- verify that the top-ranked candidate has a defensible advantage over runners-up. + +## Required fields + +| Field | Type | Constraint | +|---|---|---| +| `comparison_id` | str | Must start with `"CMP-"` | +| `batch_id` | str | Non-empty | +| `pipeline_version` | str | Non-empty | +| `comparison_date` | str | ISO 8601 `YYYY-MM-DD` | +| `candidate_ids` | list[str] | ≥ `MINIMUM_CANDIDATES` (2) entries | +| `comparison_criteria` | list[str] | ≥ `MINIMUM_CRITERIA` (2) entries | +| `top_candidate_id` | str | Must be in `candidate_ids` | +| `top_candidate_rationale` | str | Non-empty, ≤ 500 characters | +| `evidence_level` | int | 1 – 6 | +| `reviewer` | str | Non-empty | +| `dry_lab_only` | bool | Must be `True` | + +## Minimum requirements + +- At least 2 candidates must be compared (`MINIMUM_CANDIDATES = 2`). +- At least 2 comparison criteria must be stated (`MINIMUM_CRITERIA = 2`). +- The top candidate must appear in the `candidate_ids` list. + +## Valid evidence levels + +1 – 6 (from PROOF_LADDER.md) + +## Warnings + +- `evidence_level <= 2` — warning: low evidence; comparison may not be reliable. +- `len(candidate_ids) > 10` — warning: large candidate set; consider splitting into sub-groups. +- `len(comparison_criteria) < 3` — warning: fewer than 3 criteria; comparison may lack depth. + +## How to validate + +```bash +make multi-candidate-comparison-check +# or +openamp-foundry multi-candidate-comparison-check \ + --entry-json '{"comparison_id":"CMP-001","batch_id":"BATCH-001","pipeline_version":"0.8.7","comparison_date":"2026-07-09","candidate_ids":["AMP-001","AMP-002","AMP-003"],"comparison_criteria":["predicted_mic","hemolysis_fraction","selectivity_index"],"top_candidate_id":"AMP-001","top_candidate_rationale":"Highest selectivity index with lowest hemolysis fraction.","evidence_level":4,"reviewer":"alice","dry_lab_only":true}' \ + --format text +``` + +## Honest-use boundary + +Multi-candidate comparisons are dry-lab outputs. Predicted property rankings do not +imply experimental superiority. Wet-lab results may differ substantially from predictions. diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 601e51de..4102052c 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,8 @@ # Roadmap +### v0.8.7 +- Phase L L4: multi-candidate comparison schema — validates structured side-by-side comparisons of two or more candidates for publication-ready supplementary tables. MultiCandidateComparisonEntry dataclass, validate_multi_candidate_comparison(), CLI multi-candidate-comparison-check. + ### v0.8.6 - Phase L L3: candidate summary card schema — validates publication-ready per-candidate structured summaries with sequence, evidence level, activity prediction, and safety flags. CandidateSummaryCardEntry dataclass, validate_candidate_summary_card(), CLI candidate-summary-card-check. diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index ecc8473a..b7fff97b 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -3090,3 +3090,37 @@ def _run_candidate_summary_card_check(args: argparse.Namespace) -> int: print(" Candidate summary card validated. Ready for reviewer packet.") return 0 if result.passed else 3 + + +def _run_multi_candidate_comparison_check(args: argparse.Namespace) -> int: + """Validate a multi-candidate comparison entry.""" + import json as _json + from openamp_foundry.evidence.multi_candidate_comparison import validate_multi_candidate_comparison_dict + + try: + d = _json.loads(args.entry_json) + except _json.JSONDecodeError as exc: + print(_json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"})) + return 2 + + if not isinstance(d, dict): + print(_json.dumps({"status": "error", "error": "--entry-json must be a JSON object"})) + return 2 + + result = validate_multi_candidate_comparison_dict(d) + output_format = getattr(args, "format", "text") + + if output_format == "json": + import dataclasses + print(_json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + print(f"Multi-candidate comparison {result.comparison_id} (batch {result.batch_id}, {result.candidate_count} candidates): {status}") + for e in result.errors: + print(f" ERROR: {e}") + for w in result.warnings: + print(f" WARN: {w}") + if result.passed: + print(" Multi-candidate comparison validated. Ready for supplementary table.") + + return 0 if result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 0a0b369f..869affc9 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -15,7 +15,7 @@ from openamp_foundry.cli.commands.reports import _run_security_report_check from openamp_foundry.cli.commands.reports import _run_citation_check from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check -from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check, _run_candidate_summary_card_check +from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check, _run_candidate_summary_card_check, _run_multi_candidate_comparison_check import argparse import json @@ -2125,6 +2125,24 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Multi-candidate comparison check (Phase L L4) ───────────────── + mccc = sub.add_parser( + "multi-candidate-comparison-check", + help=( + "Validate a multi-candidate comparison. Confirms candidate list, " + "criteria, top-candidate rationale, and evidence level. Dry-lab only." + ), + ) + mccc.add_argument( + "--entry-json", type=str, required=True, + help="JSON string of a MultiCandidateComparisonEntry dict (required).", + ) + mccc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", @@ -2454,6 +2472,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "candidate-summary-card-check": return _run_candidate_summary_card_check(args) + if args.command == "multi-candidate-comparison-check": + return _run_multi_candidate_comparison_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/evidence/multi_candidate_comparison.py b/src/openamp_foundry/evidence/multi_candidate_comparison.py new file mode 100644 index 00000000..814c5021 --- /dev/null +++ b/src/openamp_foundry/evidence/multi_candidate_comparison.py @@ -0,0 +1,174 @@ +"""Multi-candidate comparison schema (Phase L L4). + +Validates structured side-by-side comparisons of two or more +candidates for publication-ready supplementary tables. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List + +MINIMUM_CANDIDATES: int = 2 +MINIMUM_CRITERIA: int = 2 +VALID_EVIDENCE_LEVELS: set[int] = {1, 2, 3, 4, 5, 6} +MAX_RATIONALE_LENGTH: int = 500 +LARGE_CANDIDATE_SET_THRESHOLD: int = 10 +RECOMMENDED_CRITERIA: int = 3 + +_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass +class MultiCandidateComparisonEntry: + """One multi-candidate comparison record.""" + + comparison_id: str + batch_id: str + pipeline_version: str + comparison_date: str + candidate_ids: List[str] + comparison_criteria: List[str] + top_candidate_id: str + top_candidate_rationale: str + evidence_level: int + reviewer: str + dry_lab_only: bool = True + + +@dataclass +class MultiCandidateComparisonResult: + """Validation result for a MultiCandidateComparisonEntry.""" + + comparison_id: str + batch_id: str + candidate_count: int + passed: bool + errors: List[str] + warnings: List[str] + dry_lab_only: bool = True + + +def validate_multi_candidate_comparison( + entry: MultiCandidateComparisonEntry, +) -> MultiCandidateComparisonResult: + """Validate a MultiCandidateComparisonEntry. Returns a MultiCandidateComparisonResult.""" + errors: List[str] = [] + warnings: List[str] = [] + + if not entry.comparison_id.startswith("CMP-"): + errors.append("comparison_id must start with 'CMP-'") + + if not entry.batch_id: + errors.append("batch_id must not be empty") + + if not entry.pipeline_version: + errors.append("pipeline_version must not be empty") + + if not _DATE_RE.match(entry.comparison_date): + errors.append("comparison_date must be YYYY-MM-DD") + + if len(entry.candidate_ids) < MINIMUM_CANDIDATES: + errors.append( + f"candidate_ids must have at least {MINIMUM_CANDIDATES} entries" + ) + + if len(entry.comparison_criteria) < MINIMUM_CRITERIA: + errors.append( + f"comparison_criteria must have at least {MINIMUM_CRITERIA} entries" + ) + + if entry.top_candidate_id not in entry.candidate_ids: + errors.append( + f"top_candidate_id '{entry.top_candidate_id}' must be in candidate_ids" + ) + + if not entry.top_candidate_rationale: + errors.append("top_candidate_rationale must not be empty") + elif len(entry.top_candidate_rationale) > MAX_RATIONALE_LENGTH: + errors.append( + f"top_candidate_rationale exceeds {MAX_RATIONALE_LENGTH} characters " + f"({len(entry.top_candidate_rationale)} chars)" + ) + + if entry.evidence_level not in VALID_EVIDENCE_LEVELS: + errors.append( + f"evidence_level {entry.evidence_level} is not valid; " + f"must be one of {sorted(VALID_EVIDENCE_LEVELS)}" + ) + + if not entry.reviewer: + errors.append("reviewer must not be empty") + + if entry.dry_lab_only is not True: + errors.append("dry_lab_only must be True") + + if not errors: + if entry.evidence_level <= 2: + warnings.append( + f"evidence_level {entry.evidence_level} is low (≤2); " + "comparison may not be reliable." + ) + + if len(entry.candidate_ids) > LARGE_CANDIDATE_SET_THRESHOLD: + warnings.append( + f"Large candidate set ({len(entry.candidate_ids)} candidates); " + "consider splitting into sub-groups for clarity." + ) + + if len(entry.comparison_criteria) < RECOMMENDED_CRITERIA: + warnings.append( + f"Only {len(entry.comparison_criteria)} comparison criterion/criteria; " + f"consider adding more (recommended: {RECOMMENDED_CRITERIA}+) for depth." + ) + + return MultiCandidateComparisonResult( + comparison_id=entry.comparison_id, + batch_id=entry.batch_id, + candidate_count=len(entry.candidate_ids), + passed=len(errors) == 0, + errors=errors, + warnings=warnings, + ) + + +def validate_multi_candidate_comparison_dict(d: dict) -> MultiCandidateComparisonResult: + """Validate a dict representation of a MultiCandidateComparisonEntry.""" + required = [ + "comparison_id", + "batch_id", + "pipeline_version", + "comparison_date", + "candidate_ids", + "comparison_criteria", + "top_candidate_id", + "top_candidate_rationale", + "evidence_level", + "reviewer", + ] + for key in required: + if key not in d: + return MultiCandidateComparisonResult( + comparison_id=d.get("comparison_id", ""), + batch_id=d.get("batch_id", ""), + candidate_count=0, + passed=False, + errors=[f"missing required field: {key}"], + warnings=[], + ) + + entry = MultiCandidateComparisonEntry( + comparison_id=d["comparison_id"], + batch_id=d["batch_id"], + pipeline_version=d["pipeline_version"], + comparison_date=d["comparison_date"], + candidate_ids=d["candidate_ids"], + comparison_criteria=d["comparison_criteria"], + top_candidate_id=d["top_candidate_id"], + top_candidate_rationale=d["top_candidate_rationale"], + evidence_level=d["evidence_level"], + reviewer=d["reviewer"], + dry_lab_only=d.get("dry_lab_only", True), + ) + return validate_multi_candidate_comparison(entry) diff --git a/tests/evidence/test_multi_candidate_comparison.py b/tests/evidence/test_multi_candidate_comparison.py new file mode 100644 index 00000000..80fcb598 --- /dev/null +++ b/tests/evidence/test_multi_candidate_comparison.py @@ -0,0 +1,279 @@ +"""Tests for multi-candidate comparison schema (Phase L L4).""" + +import pytest +from openamp_foundry.evidence.multi_candidate_comparison import ( + LARGE_CANDIDATE_SET_THRESHOLD, + MAX_RATIONALE_LENGTH, + MINIMUM_CANDIDATES, + MINIMUM_CRITERIA, + RECOMMENDED_CRITERIA, + VALID_EVIDENCE_LEVELS, + MultiCandidateComparisonEntry, + MultiCandidateComparisonResult, + validate_multi_candidate_comparison, + validate_multi_candidate_comparison_dict, +) + + +def _valid_entry(**kwargs) -> MultiCandidateComparisonEntry: + defaults = dict( + comparison_id="CMP-001", + batch_id="BATCH-001", + pipeline_version="0.8.7", + comparison_date="2026-07-09", + candidate_ids=["AMP-001", "AMP-002", "AMP-003"], + comparison_criteria=["predicted_mic", "hemolysis_fraction", "selectivity_index"], + top_candidate_id="AMP-001", + top_candidate_rationale="Highest selectivity index with lowest hemolysis fraction.", + evidence_level=4, + reviewer="alice", + dry_lab_only=True, + ) + defaults.update(kwargs) + return MultiCandidateComparisonEntry(**defaults) + + +# ── Constants ──────────────────────────────────────────────────────────────── + + +def test_minimum_candidates_is_two(): + assert MINIMUM_CANDIDATES == 2 + + +def test_minimum_criteria_is_two(): + assert MINIMUM_CRITERIA == 2 + + +def test_recommended_criteria_is_three(): + assert RECOMMENDED_CRITERIA == 3 + + +def test_max_rationale_length_is_500(): + assert MAX_RATIONALE_LENGTH == 500 + + +def test_large_candidate_set_threshold_is_ten(): + assert LARGE_CANDIDATE_SET_THRESHOLD == 10 + + +# ── Valid entry ─────────────────────────────────────────────────────────────── + + +def test_valid_entry_passes(): + result = validate_multi_candidate_comparison(_valid_entry()) + assert result.passed + assert result.errors == [] + + +def test_result_dry_lab_only_true(): + result = validate_multi_candidate_comparison(_valid_entry()) + assert result.dry_lab_only is True + + +def test_result_candidate_count(): + result = validate_multi_candidate_comparison(_valid_entry()) + assert result.candidate_count == 3 + + +def test_result_fields_match(): + result = validate_multi_candidate_comparison(_valid_entry()) + assert result.comparison_id == "CMP-001" + assert result.batch_id == "BATCH-001" + + +def test_valid_exactly_two_candidates(): + result = validate_multi_candidate_comparison( + _valid_entry( + candidate_ids=["AMP-001", "AMP-002"], + top_candidate_id="AMP-001", + ) + ) + assert result.passed + + +def test_valid_all_evidence_levels(): + for level in VALID_EVIDENCE_LEVELS - {1, 2}: + result = validate_multi_candidate_comparison(_valid_entry(evidence_level=level)) + assert result.passed + + +# ── comparison_id validation ────────────────────────────────────────────────── + + +def test_comparison_id_missing_prefix_fails(): + result = validate_multi_candidate_comparison(_valid_entry(comparison_id="001")) + assert not result.passed + assert any("CMP-" in e for e in result.errors) + + +def test_comparison_id_wrong_prefix_fails(): + result = validate_multi_candidate_comparison(_valid_entry(comparison_id="BND-001")) + assert not result.passed + + +def test_comparison_id_correct_prefix_passes(): + result = validate_multi_candidate_comparison(_valid_entry(comparison_id="CMP-XYZ-99")) + assert result.passed + + +# ── date validation ─────────────────────────────────────────────────────────── + + +def test_invalid_date_fails(): + result = validate_multi_candidate_comparison(_valid_entry(comparison_date="2026/07/09")) + assert not result.passed + assert any("YYYY-MM-DD" in e for e in result.errors) + + +# ── candidate_ids validation ────────────────────────────────────────────────── + + +def test_only_one_candidate_fails(): + result = validate_multi_candidate_comparison( + _valid_entry(candidate_ids=["AMP-001"], top_candidate_id="AMP-001") + ) + assert not result.passed + assert any("at least" in e and "candidate" in e for e in result.errors) + + +def test_top_candidate_not_in_list_fails(): + result = validate_multi_candidate_comparison( + _valid_entry(top_candidate_id="AMP-999") + ) + assert not result.passed + assert any("top_candidate_id" in e for e in result.errors) + + +# ── comparison_criteria validation ──────────────────────────────────────────── + + +def test_only_one_criterion_fails(): + result = validate_multi_candidate_comparison( + _valid_entry(comparison_criteria=["predicted_mic"]) + ) + assert not result.passed + assert any("comparison_criteria" in e for e in result.errors) + + +# ── rationale validation ────────────────────────────────────────────────────── + + +def test_empty_rationale_fails(): + result = validate_multi_candidate_comparison(_valid_entry(top_candidate_rationale="")) + assert not result.passed + assert any("rationale" in e for e in result.errors) + + +def test_rationale_too_long_fails(): + result = validate_multi_candidate_comparison( + _valid_entry(top_candidate_rationale="X" * 501) + ) + assert not result.passed + assert any("500" in e for e in result.errors) + + +def test_rationale_exactly_500_passes(): + result = validate_multi_candidate_comparison( + _valid_entry(top_candidate_rationale="X" * 500) + ) + assert result.passed + + +# ── dry_lab_only constraint ─────────────────────────────────────────────────── + + +def test_dry_lab_only_false_fails(): + result = validate_multi_candidate_comparison(_valid_entry(dry_lab_only=False)) + assert not result.passed + assert any("dry_lab_only" in e for e in result.errors) + + +# ── warnings ───────────────────────────────────────────────────────────────── + + +def test_low_evidence_level_warns(): + result = validate_multi_candidate_comparison(_valid_entry(evidence_level=2)) + assert result.passed + assert any("low" in w or "reliable" in w for w in result.warnings) + + +def test_large_candidate_set_warns(): + result = validate_multi_candidate_comparison( + _valid_entry( + candidate_ids=[f"AMP-{i:03d}" for i in range(11)], + top_candidate_id="AMP-000", + ) + ) + assert result.passed + assert any("Large" in w or "large" in w for w in result.warnings) + + +def test_few_criteria_warns(): + result = validate_multi_candidate_comparison( + _valid_entry(comparison_criteria=["predicted_mic", "hemolysis_fraction"]) + ) + assert result.passed + assert any("criteria" in w or "criterion" in w for w in result.warnings) + + +def test_full_comparison_no_warnings(): + result = validate_multi_candidate_comparison(_valid_entry()) + assert result.passed + assert result.warnings == [] + + +# ── dict interface ──────────────────────────────────────────────────────────── + + +def test_dict_valid_passes(): + d = dict( + comparison_id="CMP-D01", + batch_id="BATCH-D01", + pipeline_version="0.8.7", + comparison_date="2026-07-09", + candidate_ids=["AMP-001", "AMP-002"], + comparison_criteria=["predicted_mic", "selectivity_index"], + top_candidate_id="AMP-001", + top_candidate_rationale="Better selectivity.", + evidence_level=4, + reviewer="alice", + dry_lab_only=True, + ) + result = validate_multi_candidate_comparison_dict(d) + assert result.passed + + +def test_dict_missing_field_fails(): + d = dict( + comparison_id="CMP-D02", + batch_id="BATCH-D02", + pipeline_version="0.8.7", + comparison_date="2026-07-09", + candidate_ids=["AMP-001", "AMP-002"], + comparison_criteria=["predicted_mic", "selectivity_index"], + top_candidate_id="AMP-001", + top_candidate_rationale="Better.", + evidence_level=4, + # missing reviewer + ) + result = validate_multi_candidate_comparison_dict(d) + assert not result.passed + assert any("reviewer" in e for e in result.errors) + + +def test_dict_dry_lab_only_defaults_true(): + d = dict( + comparison_id="CMP-D03", + batch_id="BATCH-D03", + pipeline_version="0.8.7", + comparison_date="2026-07-09", + candidate_ids=["AMP-001", "AMP-002"], + comparison_criteria=["predicted_mic", "selectivity_index"], + top_candidate_id="AMP-001", + top_candidate_rationale="Better.", + evidence_level=4, + reviewer="alice", + ) + result = validate_multi_candidate_comparison_dict(d) + assert result.passed + assert result.dry_lab_only is True diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index bce68603..5a4a0f0a 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3946 +BASELINE = 3984 TOLERANCE = 0.05 # allow 5% fluctuation