diff --git a/docs/evidence/DOMAIN_REVIEW_OUTCOME_GUIDE.md b/docs/evidence/DOMAIN_REVIEW_OUTCOME_GUIDE.md new file mode 100644 index 00000000..571adb72 --- /dev/null +++ b/docs/evidence/DOMAIN_REVIEW_OUTCOME_GUIDE.md @@ -0,0 +1,83 @@ +# Domain Review Outcome Guide + +## Purpose + +A `DomainReviewOutcome` (DRO-) records the formal verdict of a domain expert after +reviewing a `PilotEvidencePackage`. It uses a controlled taxonomy of expert domains +and outcome verdicts, making review decisions structured, auditable, and comparable +across multiple reviewers. It links to the preceding `ReviewerQuestionnaire` (RVQ-). + +## Chain in Phase E + +``` +ESC- (external sharing) → RVQ- (reviewer questionnaire) → DRO- (expert verdict) +``` + +## Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `dro_id` | str | Yes | Unique ID, must start with `DRO-` | +| `pipeline_version` | str | Yes | Pipeline version being reviewed | +| `pep_id` | str | Yes | Which PEP was reviewed (must start with `PEP-`) | +| `rvq_id` | str | Yes | Preceding questionnaire (must start with `RVQ-`) | +| `reviewer_token` | str | Yes | Anonymized reviewer ID — no PII | +| `review_domain` | str | Yes | Expert domain (controlled vocabulary) | +| `review_date` | str | Yes | ISO YYYY-MM-DD date of verdict | +| `outcome_verdict` | str | Yes | Expert verdict (controlled vocabulary) | +| `outcome_confidence` | str | Yes | `high`, `medium`, or `low` | +| `outcome_rationale` | str | Yes | Rationale (max 400 chars) | +| `dry_lab_only` | bool | No | Default `True` | + +## Expert domains + +| Domain | When to use | +|--------|-------------| +| `antimicrobial_activity` | Reviewer assessing predicted AMP activity | +| `toxicology` | Reviewer assessing cytotoxicity/hemolysis claims | +| `structural_biology` | Reviewer assessing structural predictions | +| `clinical_microbiology` | Reviewer assessing clinical relevance | +| `computational_chemistry` | Reviewer assessing computational methods | +| `general_biomedical` | General biomedical domain expertise | + +## Outcome verdicts + +| Verdict | Meaning | +|---------|---------| +| `approve` | Package is sufficient for proceeding | +| `reject` | Package is not sufficient in current form | +| `conditional_approve` | Approve if specific conditions are met (document in rationale) | +| `request_revision` | Package needs specific revisions before re-review | +| `insufficient_data` | Cannot reach a verdict — more data required | + +## Validation rules + +1. `dro_id` must start with `DRO-` +2. `pep_id` must start with `PEP-` +3. `rvq_id` must start with `RVQ-` +4. `pipeline_version`, `reviewer_token` must not be empty +5. `review_domain` must be in the controlled vocabulary +6. `review_date` must be ISO format `YYYY-MM-DD` +7. `outcome_verdict` must be in the controlled vocabulary +8. `outcome_confidence` must be `high`, `medium`, or `low` +9. `outcome_rationale` at most 400 characters + +## Warnings + +- `conditional_approve` with no rationale — document the conditions +- `reject` with `low` confidence — may need additional reviewer +- `insufficient_data` with no rationale — document what is missing + +## Honest boundaries + +- A DRO- record reflects one expert's judgment; it does not authorize synthesis. +- `approve` does not imply biological validation — it is a dry-lab package assessment only. +- `reviewer_token` must never contain PII. + +## CLI usage + +```bash +openamp-foundry domain-review-outcome-check \ + --entry-json '{"dro_id":"DRO-001","pipeline_version":"v0.10.13","pep_id":"PEP-001","rvq_id":"RVQ-001","reviewer_token":"REV-A","review_domain":"antimicrobial_activity","review_date":"2026-07-10","outcome_verdict":"approve","outcome_confidence":"high","outcome_rationale":"Evidence base is solid."}' \ + --format text +``` diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index b7a0c408..91008146 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-10 (Phase E E3 — reviewer questionnaire — v0.10.13) +> **Last updated:** 2026-07-10 (Phase E E9 — domain review outcome — v0.10.14) +> **New in v0.10.14 (Phase E E9):** DomainReviewOutcome — controlled taxonomy expert verdict on a PEP (11 fields, 9 validation rules, 3 warnings, 63 tests, BASELINE 5414→5477); closes ESC→RVQ→DRO review chain > **New in v0.10.13 (Phase E E3):** ReviewerQuestionnaire — structured external review feedback with Likert clarity ratings, synthesis recommendation, and comments (13 fields, 10 validation rules, 4 warnings, 63 tests, BASELINE 5351→5414) > **New in v0.10.12 (Phase F F4):** FailedCandidateBatchReport — batch-level failure summary linking RJR- and NAS- with failure rate consistency check (13 fields, 11 validation rules, 4 warnings, 63 tests, BASELINE 5288→5351) > **New in v0.10.11 (Phase F F2):** NegativeResultArchiveSummary — validates batch of NRR- entries is complete before archiving (11 fields, 9 validation rules, 3 warnings, 63 tests, BASELINE 5225→5288) diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index f2df1fa3..8a67de48 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -100,7 +100,7 @@ Make qualified external review easier and safer. | E6 | Add packet generator CLI. | Reduces manual packaging errors. | C/D | | E7 | Add packet validator CLI. | Review readiness becomes testable. | C/D | | E8 | Add release-summary generator that strips restricted fields. | Safer public summaries. | D | -| E9 | Add domain-review outcome taxonomy. | Rejection becomes structured. | B/C | +| E9 | Add domain review outcome schema (complete). | Structured expert verdict on a PEP with controlled taxonomy of domains and outcomes; closes ESC→RVQ→DRO review chain. | B/C | | E10 | Add expert-review example with mock/toy candidates only. | Safe demonstration. | B/C | ## Phase F — Negative-result infrastructure diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 08f6f1b6..774d17ce 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1285,6 +1285,51 @@ def _run_validate_simulation_result(args: argparse.Namespace) -> int: return 3 if batch_result["any_invalid"] else 0 +def _run_domain_review_outcome_check(args): + from openamp_foundry.evidence.domain_review_outcome import ( + validate_domain_review_outcome_dict, + ) + import json + import sys + + data = json.loads(args.entry_json) + result = validate_domain_review_outcome_dict(data) + + if args.format == "json": + out = { + "dro_id": result.dro_id, + "pep_id": result.pep_id, + "rvq_id": result.rvq_id, + "review_domain": result.review_domain, + "outcome_verdict": result.outcome_verdict, + "outcome_confidence": result.outcome_confidence, + "passed": result.passed, + "errors": result.errors, + "warnings": result.warnings, + "dry_lab_only": result.dry_lab_only, + } + print(json.dumps(out, indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + print(f"Domain Review Outcome: {status}") + print(f" DRO ID: {result.dro_id}") + print(f" PEP ID: {result.pep_id}") + print(f" RVQ ID: {result.rvq_id}") + print(f" Domain: {result.review_domain}") + print(f" Verdict: {result.outcome_verdict}") + print(f" Confidence: {result.outcome_confidence}") + if result.errors: + print(" Errors:") + for e in result.errors: + print(f" - {e}") + if result.warnings: + print(" Warnings:") + for w in result.warnings: + print(f" - {w}") + + sys.exit(0 if result.passed else 1) + + def _run_simulation_registry(args: argparse.Namespace) -> int: """Display simulation module registry information.""" from openamp_foundry.simulation.module_registry import ( diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 587922c8..c4260ec6 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -23,6 +23,7 @@ _run_negative_result_archive_check, _run_failed_candidate_batch_report_check, _run_reviewer_questionnaire_check, + _run_domain_review_outcome_check, _run_hypothesis_outcome_check, _run_baseline_comparison_check, _run_negative_result_check, _run_experiment_priority_check, _run_calibration_performance_check, _run_prediction_drift_check, _run_calibration_improvement_check, @@ -2328,6 +2329,15 @@ def build_parser() -> argparse.ArgumentParser: "--format", choices=["text", "json"], default="text" ) + domain_review_outcome_parser = sub.add_parser( + "domain-review-outcome-check", + help="Validate a DomainReviewOutcome (DRO-).", + ) + domain_review_outcome_parser.add_argument("--entry-json", required=True) + domain_review_outcome_parser.add_argument( + "--format", choices=["text", "json"], default="text" + ) + # ── Hypothesis outcome check (Phase N N2) ──────────────────────── hypothesis_outcome_parser = sub.add_parser( "hypothesis-outcome-check", @@ -2837,6 +2847,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "reviewer-questionnaire-check": _run_reviewer_questionnaire_check(args) + if args.command == "domain-review-outcome-check": + _run_domain_review_outcome_check(args) + if args.command == "hypothesis-outcome-check": return _run_hypothesis_outcome_check(args) diff --git a/src/openamp_foundry/evidence/domain_review_outcome.py b/src/openamp_foundry/evidence/domain_review_outcome.py new file mode 100644 index 00000000..1df68015 --- /dev/null +++ b/src/openamp_foundry/evidence/domain_review_outcome.py @@ -0,0 +1,200 @@ +"""Domain review outcome schema — Phase E E9. + +Records the final verdict of a domain expert reviewer on a PilotEvidencePackage. +Uses a controlled taxonomy of expert domains and outcome verdicts, making expert +review decisions structured, auditable, and comparable across reviewers. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List + +DRO_PREFIX = "DRO-" +RVQ_PREFIX = "RVQ-" +PEP_PREFIX = "PEP-" + +VALID_REVIEW_DOMAINS = frozenset({ + "antimicrobial_activity", + "toxicology", + "structural_biology", + "clinical_microbiology", + "computational_chemistry", + "general_biomedical", +}) + +VALID_OUTCOME_VERDICTS = frozenset({ + "approve", + "reject", + "conditional_approve", + "request_revision", + "insufficient_data", +}) + +VALID_OUTCOME_CONFIDENCES = frozenset({ + "high", + "medium", + "low", +}) + +RATIONALE_MAX_LENGTH = 400 + + +@dataclass +class DomainReviewOutcome: + """Formal expert verdict on a PilotEvidencePackage.""" + + dro_id: str + pipeline_version: str + pep_id: str # must start with "PEP-" + rvq_id: str # must start with "RVQ-" — the questionnaire that preceded this verdict + reviewer_token: str # anonymized reviewer ID, no PII + review_domain: str # controlled vocabulary + review_date: str # ISO YYYY-MM-DD + outcome_verdict: str # controlled vocabulary + outcome_confidence: str # {high, medium, low} + outcome_rationale: str # max 400 chars + dry_lab_only: bool = True + + +@dataclass +class DomainReviewOutcomeResult: + dro_id: str + pep_id: str + rvq_id: str + review_domain: str + outcome_verdict: str + outcome_confidence: str + passed: bool + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + dry_lab_only: bool = True + + +def validate_domain_review_outcome( + entry: DomainReviewOutcome, +) -> DomainReviewOutcomeResult: + errors: List[str] = [] + warnings: List[str] = [] + + # Rule 1: DRO- prefix + if not entry.dro_id.startswith(DRO_PREFIX): + errors.append( + f"dro_id must start with '{DRO_PREFIX}', got '{entry.dro_id}'" + ) + + # Rule 2: PEP- prefix + if not entry.pep_id.startswith(PEP_PREFIX): + errors.append( + f"pep_id must start with '{PEP_PREFIX}', got '{entry.pep_id}'" + ) + + # Rule 3: RVQ- prefix + if not entry.rvq_id.startswith(RVQ_PREFIX): + errors.append( + f"rvq_id must start with '{RVQ_PREFIX}', got '{entry.rvq_id}'" + ) + + # Rule 4: non-empty required strings + for fname, val in [ + ("pipeline_version", entry.pipeline_version), + ("reviewer_token", entry.reviewer_token), + ]: + if not val or not val.strip(): + errors.append(f"{fname} must not be empty") + + # Rule 5: valid review domain + if entry.review_domain not in VALID_REVIEW_DOMAINS: + errors.append( + f"review_domain must be one of {sorted(VALID_REVIEW_DOMAINS)}, " + f"got '{entry.review_domain}'" + ) + + # Rule 6: ISO date + if not re.match(r"^\d{4}-\d{2}-\d{2}$", entry.review_date): + errors.append( + f"review_date must be ISO format YYYY-MM-DD, got '{entry.review_date}'" + ) + + # Rule 7: valid outcome verdict + if entry.outcome_verdict not in VALID_OUTCOME_VERDICTS: + errors.append( + f"outcome_verdict must be one of {sorted(VALID_OUTCOME_VERDICTS)}, " + f"got '{entry.outcome_verdict}'" + ) + + # Rule 8: valid outcome confidence + if entry.outcome_confidence not in VALID_OUTCOME_CONFIDENCES: + errors.append( + f"outcome_confidence must be one of {sorted(VALID_OUTCOME_CONFIDENCES)}, " + f"got '{entry.outcome_confidence}'" + ) + + # Rule 9: rationale length + if len(entry.outcome_rationale) > RATIONALE_MAX_LENGTH: + errors.append( + f"outcome_rationale must be at most {RATIONALE_MAX_LENGTH} characters, " + f"got {len(entry.outcome_rationale)}" + ) + + # Warning 1: conditional_approve without rationale + if ( + entry.outcome_verdict == "conditional_approve" + and not entry.outcome_rationale.strip() + ): + warnings.append( + "outcome_verdict is 'conditional_approve' but outcome_rationale is empty" + " — document the conditions required for approval" + ) + + # Warning 2: reject with low confidence + if ( + entry.outcome_verdict == "reject" + and entry.outcome_confidence == "low" + ): + warnings.append( + "outcome_verdict is 'reject' with outcome_confidence='low'" + " — a low-confidence rejection may need additional expert review" + ) + + # Warning 3: insufficient_data without rationale + if ( + entry.outcome_verdict == "insufficient_data" + and not entry.outcome_rationale.strip() + ): + warnings.append( + "outcome_verdict is 'insufficient_data' but outcome_rationale is empty" + " — document what data is missing" + ) + + passed = len(errors) == 0 + return DomainReviewOutcomeResult( + dro_id=entry.dro_id, + pep_id=entry.pep_id, + rvq_id=entry.rvq_id, + review_domain=entry.review_domain, + outcome_verdict=entry.outcome_verdict, + outcome_confidence=entry.outcome_confidence, + passed=passed, + errors=errors, + warnings=warnings, + dry_lab_only=entry.dry_lab_only, + ) + + +def validate_domain_review_outcome_dict(data: dict) -> DomainReviewOutcomeResult: + entry = DomainReviewOutcome( + dro_id=data.get("dro_id", ""), + pipeline_version=data.get("pipeline_version", ""), + pep_id=data.get("pep_id", ""), + rvq_id=data.get("rvq_id", ""), + reviewer_token=data.get("reviewer_token", ""), + review_domain=data.get("review_domain", ""), + review_date=data.get("review_date", ""), + outcome_verdict=data.get("outcome_verdict", ""), + outcome_confidence=data.get("outcome_confidence", ""), + outcome_rationale=data.get("outcome_rationale", ""), + dry_lab_only=bool(data.get("dry_lab_only", True)), + ) + return validate_domain_review_outcome(entry) diff --git a/tests/evidence/test_domain_review_outcome.py b/tests/evidence/test_domain_review_outcome.py new file mode 100644 index 00000000..67b873d2 --- /dev/null +++ b/tests/evidence/test_domain_review_outcome.py @@ -0,0 +1,455 @@ +"""Tests for DomainReviewOutcome schema — Phase E E9.""" + +from __future__ import annotations + +import pytest + +from openamp_foundry.evidence.domain_review_outcome import ( + DomainReviewOutcome, + DomainReviewOutcomeResult, + validate_domain_review_outcome, + validate_domain_review_outcome_dict, +) + + +def _valid_entry(**overrides) -> DomainReviewOutcome: + defaults = dict( + dro_id="DRO-001", + pipeline_version="v0.10.13", + pep_id="PEP-001", + rvq_id="RVQ-001", + reviewer_token="REV-TOKEN-A", + review_domain="antimicrobial_activity", + review_date="2026-07-10", + outcome_verdict="approve", + outcome_confidence="high", + outcome_rationale="Activity predictions are well-supported by the evidence.", + dry_lab_only=True, + ) + defaults.update(overrides) + return DomainReviewOutcome(**defaults) + + +# ── 1. DRO ID validation ────────────────────────────────────────────────────── + +class TestDroIdValidation: + def test_valid_dro_prefix(self): + result = validate_domain_review_outcome(_valid_entry()) + assert result.passed + + def test_wrong_prefix_fails(self): + result = validate_domain_review_outcome(_valid_entry(dro_id="RVQ-001")) + assert not result.passed + assert any("dro_id must start with 'DRO-'" in e for e in result.errors) + + def test_empty_dro_id_fails(self): + result = validate_domain_review_outcome(_valid_entry(dro_id="")) + assert not result.passed + + def test_lowercase_prefix_fails(self): + result = validate_domain_review_outcome(_valid_entry(dro_id="dro-001")) + assert not result.passed + + def test_dro_long_id_passes(self): + result = validate_domain_review_outcome(_valid_entry(dro_id="DRO-20260710-001")) + assert result.passed + + def test_dro_id_in_result(self): + result = validate_domain_review_outcome(_valid_entry(dro_id="DRO-XYZ")) + assert result.dro_id == "DRO-XYZ" + + def test_no_prefix_fails(self): + result = validate_domain_review_outcome(_valid_entry(dro_id="001")) + assert not result.passed + + +# ── 2. PEP ID validation ────────────────────────────────────────────────────── + +class TestPepIdValidation: + def test_valid_pep_prefix(self): + result = validate_domain_review_outcome(_valid_entry()) + assert result.passed + + def test_wrong_pep_prefix_fails(self): + result = validate_domain_review_outcome(_valid_entry(pep_id="RVQ-001")) + assert not result.passed + assert any("pep_id must start with 'PEP-'" in e for e in result.errors) + + def test_empty_pep_id_fails(self): + result = validate_domain_review_outcome(_valid_entry(pep_id="")) + assert not result.passed + + def test_pep_id_in_result(self): + result = validate_domain_review_outcome(_valid_entry(pep_id="PEP-999")) + assert result.pep_id == "PEP-999" + + def test_pep_long_id_passes(self): + result = validate_domain_review_outcome(_valid_entry(pep_id="PEP-2026-001")) + assert result.passed + + +# ── 3. RVQ ID validation ────────────────────────────────────────────────────── + +class TestRvqIdValidation: + def test_valid_rvq_prefix(self): + result = validate_domain_review_outcome(_valid_entry()) + assert result.passed + + def test_wrong_rvq_prefix_fails(self): + result = validate_domain_review_outcome(_valid_entry(rvq_id="PEP-001")) + assert not result.passed + assert any("rvq_id must start with 'RVQ-'" in e for e in result.errors) + + def test_empty_rvq_id_fails(self): + result = validate_domain_review_outcome(_valid_entry(rvq_id="")) + assert not result.passed + + def test_rvq_id_in_result(self): + result = validate_domain_review_outcome(_valid_entry(rvq_id="RVQ-999")) + assert result.rvq_id == "RVQ-999" + + def test_rvq_long_id_passes(self): + result = validate_domain_review_outcome(_valid_entry(rvq_id="RVQ-2026-001")) + assert result.passed + + +# ── 4. Required fields ──────────────────────────────────────────────────────── + +class TestRequiredFields: + def test_all_valid_passes(self): + result = validate_domain_review_outcome(_valid_entry()) + assert result.passed + + def test_empty_pipeline_version_fails(self): + result = validate_domain_review_outcome(_valid_entry(pipeline_version="")) + assert not result.passed + assert any("pipeline_version must not be empty" in e for e in result.errors) + + def test_whitespace_pipeline_fails(self): + result = validate_domain_review_outcome(_valid_entry(pipeline_version=" ")) + assert not result.passed + + def test_empty_reviewer_token_fails(self): + result = validate_domain_review_outcome(_valid_entry(reviewer_token="")) + assert not result.passed + assert any("reviewer_token must not be empty" in e for e in result.errors) + + def test_whitespace_reviewer_token_fails(self): + result = validate_domain_review_outcome(_valid_entry(reviewer_token=" ")) + assert not result.passed + + def test_multiple_empty_fields_all_reported(self): + result = validate_domain_review_outcome( + _valid_entry(pipeline_version="", reviewer_token="") + ) + assert not result.passed + assert len([e for e in result.errors if "must not be empty" in e]) == 2 + + +# ── 5. Review domain validation ─────────────────────────────────────────────── + +class TestReviewDomainValidation: + def test_antimicrobial_activity_passes(self): + result = validate_domain_review_outcome( + _valid_entry(review_domain="antimicrobial_activity") + ) + assert result.passed + + def test_toxicology_passes(self): + result = validate_domain_review_outcome(_valid_entry(review_domain="toxicology")) + assert result.passed + + def test_structural_biology_passes(self): + result = validate_domain_review_outcome( + _valid_entry(review_domain="structural_biology") + ) + assert result.passed + + def test_clinical_microbiology_passes(self): + result = validate_domain_review_outcome( + _valid_entry(review_domain="clinical_microbiology") + ) + assert result.passed + + def test_computational_chemistry_passes(self): + result = validate_domain_review_outcome( + _valid_entry(review_domain="computational_chemistry") + ) + assert result.passed + + def test_general_biomedical_passes(self): + result = validate_domain_review_outcome( + _valid_entry(review_domain="general_biomedical") + ) + assert result.passed + + def test_invalid_domain_fails(self): + result = validate_domain_review_outcome(_valid_entry(review_domain="physics")) + assert not result.passed + assert any("review_domain must be one of" in e for e in result.errors) + + def test_empty_domain_fails(self): + result = validate_domain_review_outcome(_valid_entry(review_domain="")) + assert not result.passed + + def test_domain_in_result(self): + result = validate_domain_review_outcome( + _valid_entry(review_domain="toxicology") + ) + assert result.review_domain == "toxicology" + + +# ── 6. Review date ──────────────────────────────────────────────────────────── + +class TestReviewDate: + def test_valid_date(self): + result = validate_domain_review_outcome(_valid_entry(review_date="2026-07-10")) + assert result.passed + + def test_date_without_dashes_fails(self): + result = validate_domain_review_outcome(_valid_entry(review_date="20260710")) + assert not result.passed + assert any("review_date must be ISO format" in e for e in result.errors) + + def test_slash_date_fails(self): + result = validate_domain_review_outcome(_valid_entry(review_date="2026/07/10")) + assert not result.passed + + def test_empty_date_fails(self): + result = validate_domain_review_outcome(_valid_entry(review_date="")) + assert not result.passed + + def test_another_valid_date(self): + result = validate_domain_review_outcome(_valid_entry(review_date="2025-06-01")) + assert result.passed + + +# ── 7. Outcome verdict validation ───────────────────────────────────────────── + +class TestOutcomeVerdictValidation: + def test_approve_passes(self): + result = validate_domain_review_outcome(_valid_entry(outcome_verdict="approve")) + assert result.passed + + def test_reject_passes(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_verdict="reject", outcome_confidence="high") + ) + assert result.passed + + def test_conditional_approve_with_rationale_passes(self): + result = validate_domain_review_outcome( + _valid_entry( + outcome_verdict="conditional_approve", + outcome_rationale="Approve if baseline is updated.", + ) + ) + assert result.passed + + def test_request_revision_passes(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_verdict="request_revision") + ) + assert result.passed + + def test_insufficient_data_with_rationale_passes(self): + result = validate_domain_review_outcome( + _valid_entry( + outcome_verdict="insufficient_data", + outcome_rationale="Need full BCM report.", + ) + ) + assert result.passed + + def test_invalid_verdict_fails(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_verdict="maybe") + ) + assert not result.passed + assert any("outcome_verdict must be one of" in e for e in result.errors) + + def test_empty_verdict_fails(self): + result = validate_domain_review_outcome(_valid_entry(outcome_verdict="")) + assert not result.passed + + def test_verdict_in_result(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_verdict="request_revision") + ) + assert result.outcome_verdict == "request_revision" + + +# ── 8. Outcome confidence validation ────────────────────────────────────────── + +class TestOutcomeConfidenceValidation: + def test_high_passes(self): + result = validate_domain_review_outcome(_valid_entry(outcome_confidence="high")) + assert result.passed + + def test_medium_passes(self): + result = validate_domain_review_outcome(_valid_entry(outcome_confidence="medium")) + assert result.passed + + def test_low_passes(self): + result = validate_domain_review_outcome(_valid_entry(outcome_confidence="low")) + assert result.passed + + def test_invalid_confidence_fails(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_confidence="very_high") + ) + assert not result.passed + assert any("outcome_confidence must be one of" in e for e in result.errors) + + def test_empty_confidence_fails(self): + result = validate_domain_review_outcome(_valid_entry(outcome_confidence="")) + assert not result.passed + + def test_confidence_in_result(self): + result = validate_domain_review_outcome(_valid_entry(outcome_confidence="medium")) + assert result.outcome_confidence == "medium" + + def test_reject_low_confidence_warns(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_verdict="reject", outcome_confidence="low") + ) + assert result.passed + assert any("low-confidence rejection" in w for w in result.warnings) + + +# ── 9. Warnings ─────────────────────────────────────────────────────────────── + +class TestWarnings: + def test_conditional_approve_no_rationale_warns(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_verdict="conditional_approve", outcome_rationale="") + ) + assert result.passed + assert any("conditional_approve" in w for w in result.warnings) + + def test_conditional_approve_with_rationale_no_warning(self): + result = validate_domain_review_outcome( + _valid_entry( + outcome_verdict="conditional_approve", + outcome_rationale="Need updated baseline.", + ) + ) + assert result.passed + assert not any("conditional_approve" in w for w in result.warnings) + + def test_insufficient_data_no_rationale_warns(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_verdict="insufficient_data", outcome_rationale="") + ) + assert result.passed + assert any("insufficient_data" in w for w in result.warnings) + + def test_approve_high_confidence_no_warnings(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_verdict="approve", outcome_confidence="high") + ) + assert result.passed + assert len(result.warnings) == 0 + + def test_dry_lab_only_in_result(self): + result = validate_domain_review_outcome(_valid_entry(dry_lab_only=True)) + assert result.dry_lab_only is True + + def test_rationale_length_at_limit_passes(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_rationale="x" * 400) + ) + assert result.passed + + def test_rationale_over_limit_fails(self): + result = validate_domain_review_outcome( + _valid_entry(outcome_rationale="x" * 401) + ) + assert not result.passed + assert any("outcome_rationale must be at most 400" in e for e in result.errors) + + +# ── 10. Dict-based validator ────────────────────────────────────────────────── + +class TestDictValidator: + def test_valid_dict_passes(self): + data = { + "dro_id": "DRO-001", + "pipeline_version": "v0.10.13", + "pep_id": "PEP-001", + "rvq_id": "RVQ-001", + "reviewer_token": "REV-A", + "review_domain": "antimicrobial_activity", + "review_date": "2026-07-10", + "outcome_verdict": "approve", + "outcome_confidence": "high", + "outcome_rationale": "Solid evidence base.", + "dry_lab_only": True, + } + result = validate_domain_review_outcome_dict(data) + assert result.passed + + def test_dict_wrong_prefix_fails(self): + data = { + "dro_id": "BAD-001", + "pipeline_version": "v0.10.13", + "pep_id": "PEP-001", + "rvq_id": "RVQ-001", + "reviewer_token": "REV-A", + "review_domain": "toxicology", + "review_date": "2026-07-10", + "outcome_verdict": "approve", + "outcome_confidence": "high", + "outcome_rationale": "ok", + } + result = validate_domain_review_outcome_dict(data) + assert not result.passed + + def test_dict_invalid_domain_fails(self): + data = { + "dro_id": "DRO-001", + "pipeline_version": "v0.10.13", + "pep_id": "PEP-001", + "rvq_id": "RVQ-001", + "reviewer_token": "REV-A", + "review_domain": "astrophysics", + "review_date": "2026-07-10", + "outcome_verdict": "approve", + "outcome_confidence": "high", + "outcome_rationale": "ok", + } + result = validate_domain_review_outcome_dict(data) + assert not result.passed + + def test_dict_defaults_dry_lab_true(self): + data = { + "dro_id": "DRO-001", + "pipeline_version": "v0.10.13", + "pep_id": "PEP-001", + "rvq_id": "RVQ-001", + "reviewer_token": "REV-A", + "review_domain": "clinical_microbiology", + "review_date": "2026-07-10", + "outcome_verdict": "request_revision", + "outcome_confidence": "medium", + "outcome_rationale": "Needs more data.", + } + result = validate_domain_review_outcome_dict(data) + assert result.dry_lab_only is True + + def test_dict_multiple_errors_reported(self): + data = { + "dro_id": "BAD", + "pipeline_version": "", + "pep_id": "WRONG", + "rvq_id": "WRONG", + "reviewer_token": "", + "review_domain": "unknown", + "review_date": "not-a-date", + "outcome_verdict": "invalid", + "outcome_confidence": "invalid", + "outcome_rationale": "x" * 500, + } + result = validate_domain_review_outcome_dict(data) + assert not result.passed + assert len(result.errors) >= 5 diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 274d8a56..65f28ed3 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 5414 +BASELINE = 5477 TOLERANCE = 0.05 # allow 5% fluctuation