diff --git a/docs/evidence/EXPERT_REVIEW_EXAMPLE_PACKAGE_GUIDE.md b/docs/evidence/EXPERT_REVIEW_EXAMPLE_PACKAGE_GUIDE.md new file mode 100644 index 00000000..ba6b2d0d --- /dev/null +++ b/docs/evidence/EXPERT_REVIEW_EXAMPLE_PACKAGE_GUIDE.md @@ -0,0 +1,79 @@ +# ExpertReviewExamplePackage (ERP-) Guide + +## Purpose + +`ExpertReviewExamplePackage` is a self-contained example of a complete expert-review +submission, using only toy/mock candidates. It exists so external reviewers and +partners know exactly what to expect before receiving real data. + +**This schema is for examples only.** Real review submissions use +`ReviewerQuestionnaire` (RVQ-) and `DomainReviewOutcome` (DRO-) schemas. + +## Schema Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `erp_id` | str | yes | Unique identifier, must start with `ERP-` | +| `pipeline_version` | str | yes | Pipeline version this example targets | +| `example_version` | str | yes | Version of this example (e.g. `v1.0`) | +| `creation_date` | str | yes | ISO date (YYYY-MM-DD) | +| `review_domain` | str | yes | Domain focus (controlled vocabulary) | +| `mock_candidates` | list | yes | 1-10 toy candidate summaries | +| `overall_clarity_rating` | int | yes | Likert 1-5 overall package clarity | +| `synthesis_recommendation` | str | yes | Example recommendation (controlled vocabulary) | +| `reviewer_comments` | str | yes | Example reviewer comments (≤500 chars) | +| `dry_lab_only` | bool | yes | Must be True | +| `is_example_data` | bool | yes | Must be True | +| `example_use_case` | str | yes | What this example demonstrates | +| `summary` | str | yes | Summary of the example (≤400 chars) | +| `notes` | str | no | Additional context (≤300 chars) | + +## Mock Candidate Fields + +Each entry in `mock_candidates` must have: +- `candidate_id`: must start with `MOCK-`, `TOY-`, `EXAMPLE-`, `DEMO-`, or `TEST-` +- `sequence_length`: positive integer +- `predicted_mic`: positive float (µg/mL) +- `predicted_toxicity`: `low`, `moderate`, or `high` +- `novelty_score`: float in [0.0, 1.0] +- `include_in_example`: bool (at least one must be True) + +## Controlled Vocabularies + +**review_domain (6 values):** +`antimicrobial_activity`, `toxicity_safety`, `novelty_assessment`, +`simulation_quality`, `evidence_completeness`, `experimental_design` + +**synthesis_recommendation (4 values):** +`proceed`, `proceed_with_conditions`, `defer`, `reject` + +## Validation Rules + +1. `erp_id` starts with `ERP-` +2. `pipeline_version` non-empty +3. `example_version` non-empty +4. `creation_date` is ISO date (YYYY-MM-DD) +5. `review_domain` in controlled vocabulary (6 values) +6. `mock_candidates` has 1-10 entries +7. Each candidate: ID prefix, sequence_length ≥1, mic >0, toxicity vocab, novelty [0,1] +8. At least one candidate has `include_in_example=True` +9. `overall_clarity_rating` in {1,2,3,4,5} +10. `synthesis_recommendation` in controlled vocabulary (4 values) +11. `reviewer_comments` non-empty and ≤500 chars +12. `dry_lab_only` must be True +13. `is_example_data` must be True +14. `example_use_case` non-empty +15. `summary` non-empty and ≤400 chars +16. `notes` ≤300 chars + +## Warnings + +- `overall_clarity_rating ≤ 2`: verify the example package is complete +- Excluded candidates: verify `include_in_example=False` is intentional +- `notes` empty: consider documenting example context + +## Integration + +- Real review submissions: use `ReviewerQuestionnaire` (RVQ-) +- Expert verdicts: use `DomainReviewOutcome` (DRO-) +- This schema: onboarding, CI validation, partner documentation diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 311fd777..fff51625 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -9,6 +9,14 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak ## Changelog +### v0.10.24 — Phase E E10: ExpertReviewExamplePackage schema +- Added `ExpertReviewExamplePackage` (ERP-) schema +- 14 fields: erp_id, pipeline_version, example_version, creation_date, review_domain, mock_candidates (1-10), overall_clarity_rating, synthesis_recommendation, reviewer_comments, dry_lab_only, is_example_data, example_use_case, summary, notes +- 16 validation rules; 3 warnings; 63 tests +- BASELINE 6107→6170 +- CLI: openamp-foundry expert-review-example-package-check +- Closes Phase E E10 — CI-checkable example package that cannot leak real candidates + ### v0.10.23 — Phase G G8: SyntheticBoundaryAuditRecord schema - Added `SyntheticBoundaryAuditRecord` (SBR-) schema - 14 fields: sbr_id, pipeline_version, batch_id, audit_date, evidence_source, total_candidates_checked, total_violations, violation_rate, blocked_upgrades, max_proposed_ladder_level, policy_enforced, enforcement_outcome, summary, notes diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index dbbd0207..068c4d5e 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -101,7 +101,7 @@ Make qualified external review easier and safer. | E7 | Add packet validator CLI. | Review readiness becomes testable. | C/D | | E8 | Add release-summary generator that strips restricted fields. | Safer public summaries. | D | | E9 | Add domain review outcome schema (complete). | Structured expert verdict on a PEP with controlled taxonomy of domains and outcomes; closes ESC→RVQ→DRO review chain. | B/C | -| E10 | Add expert-review example with mock/toy candidates only. | Safe demonstration. | B/C | +| E10 | Add expert-review example with mock/toy candidates only (complete). | ERP- schema: 14 fields, 16 validation rules, mock candidate ID prefix enforcement (MOCK-/TOY-/EXAMPLE-/DEMO-/TEST-), is_example_data=True and dry_lab_only=True enforced; CI-checkable template cannot accidentally leak real candidates. | B/C | ## Phase F — Negative-result infrastructure diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index a38aa97d..09eb52e7 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -3065,3 +3065,20 @@ def _run_synthetic_boundary_audit_record_check(args): print(f"ERROR: {e}", file=sys.stderr) sys.exit(1) print("OK: SyntheticBoundaryAuditRecord is valid.") + + +def _run_expert_review_example_package_check(args): + """CLI handler for expert-review-example-package-check.""" + import json, sys + from openamp_foundry.evidence.expert_review_example_package import validate_dict + data = json.load(open(args.input)) + issues = validate_dict(data) + errors = [i for i in issues if not i.startswith("WARNING:")] + warnings = [i for i in issues if i.startswith("WARNING:")] + for w in warnings: + print(w) + if errors: + for e in errors: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + print("OK: ExpertReviewExamplePackage is valid.") diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 4087fe31..7a9f65c7 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -39,6 +39,7 @@ _run_recalibration_decision_log_check, _run_recalibration_rejection_summary_check, _run_synthetic_boundary_audit_record_check, + _run_expert_review_example_package_check, ) from openamp_foundry.cli.commands.gates import _run_gate_check, _run_release_gate_check from openamp_foundry.cli.commands.reports import _run_contribution_check @@ -2543,6 +2544,10 @@ def build_parser() -> argparse.ArgumentParser: p_sbr.add_argument("input", help="Path to JSON file") p_sbr.set_defaults(func=_run_synthetic_boundary_audit_record_check) + p_erp = subparsers.add_parser("expert-review-example-package-check", help="Validate an ExpertReviewExamplePackage JSON") + p_erp.add_argument("input", help="Path to JSON file") + p_erp.set_defaults(func=_run_expert_review_example_package_check) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", @@ -2988,6 +2993,9 @@ def main(argv: list[str] | None = None) -> int: elif args.command == "recalibration-decision-log-check": _run_recalibration_decision_log_check(args) + elif args.command == "expert-review-example-package-check": + _run_expert_review_example_package_check(args) + else: parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/evidence/expert_review_example_package.py b/src/openamp_foundry/evidence/expert_review_example_package.py new file mode 100644 index 00000000..62656dbb --- /dev/null +++ b/src/openamp_foundry/evidence/expert_review_example_package.py @@ -0,0 +1,253 @@ +"""Expert review example package schema — Phase E E10. + +A self-contained example of a complete expert-review submission, using only +toy/mock candidate data clearly labeled as such. This schema: + + 1. Shows external reviewers what format to expect before they receive real data. + 2. Provides a CI-checkable template that cannot accidentally leak real candidates. + 3. Documents the full review workflow in a machine-readable, testable artifact. + +All candidate data in this schema MUST be mock/toy. Real candidates must use +ReviewerQuestionnaire (RVQ-) and DomainReviewOutcome (DRO-) schemas instead. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List + +ERP_PREFIX = "ERP-" + +VALID_REVIEW_DOMAINS = frozenset({ + "antimicrobial_activity", + "toxicity_safety", + "novelty_assessment", + "simulation_quality", + "evidence_completeness", + "experimental_design", +}) + +VALID_SYNTHESIS_RECOMMENDATIONS = frozenset({ + "proceed", + "proceed_with_conditions", + "defer", + "reject", +}) + +VALID_CLARITY_RATINGS = frozenset({1, 2, 3, 4, 5}) # Likert 1-5 + +MOCK_CANDIDATE_ID_PREFIXES = frozenset({ + "MOCK-", "TOY-", "EXAMPLE-", "DEMO-", "TEST-", +}) + +MIN_MOCK_CANDIDATES = 1 +MAX_MOCK_CANDIDATES = 10 +REVIEWER_COMMENTS_MAX_LENGTH = 500 +SUMMARY_MAX_LENGTH = 400 +NOTES_MAX_LENGTH = 300 +_ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@dataclass +class MockCandidateSummary: + """Summary of a single mock/toy candidate in an example review package.""" + candidate_id: str # must start with a mock prefix + sequence_length: int # length of mock peptide (positive int) + predicted_mic: float # mock MIC value in µg/mL (positive) + predicted_toxicity: str # one of: "low", "moderate", "high" + novelty_score: float # [0.0, 1.0] + include_in_example: bool # True = shown to reviewer in this example + + +@dataclass +class ExpertReviewExamplePackage: + erp_id: str + pipeline_version: str + example_version: str + creation_date: str + review_domain: str + mock_candidates: List[MockCandidateSummary] + overall_clarity_rating: int # Likert 1-5 + synthesis_recommendation: str + reviewer_comments: str + dry_lab_only: bool + is_example_data: bool # must be True for this schema + example_use_case: str # what this example demonstrates + summary: str + notes: str = "" + + +def _validate_mock_candidate( + candidate: MockCandidateSummary, index: int +) -> List[str]: + errors: List[str] = [] + prefix_ok = any( + candidate.candidate_id.startswith(p) for p in MOCK_CANDIDATE_ID_PREFIXES + ) + if not prefix_ok: + errors.append( + f"mock_candidates[{index}].candidate_id must start with one of " + f"{sorted(MOCK_CANDIDATE_ID_PREFIXES)}, got: {candidate.candidate_id!r}" + ) + if candidate.sequence_length < 1: + errors.append( + f"mock_candidates[{index}].sequence_length must be >= 1, " + f"got: {candidate.sequence_length}" + ) + if candidate.predicted_mic <= 0: + errors.append( + f"mock_candidates[{index}].predicted_mic must be > 0, " + f"got: {candidate.predicted_mic}" + ) + valid_toxicities = frozenset({"low", "moderate", "high"}) + if candidate.predicted_toxicity not in valid_toxicities: + errors.append( + f"mock_candidates[{index}].predicted_toxicity must be one of " + f"{sorted(valid_toxicities)}, got: {candidate.predicted_toxicity!r}" + ) + if not (0.0 <= candidate.novelty_score <= 1.0): + errors.append( + f"mock_candidates[{index}].novelty_score must be in [0.0, 1.0], " + f"got: {candidate.novelty_score}" + ) + return errors + + +def validate(pkg: ExpertReviewExamplePackage) -> List[str]: + """Return list of error strings; empty list means valid.""" + errors: List[str] = [] + + # Rule 1: ID prefix + if not pkg.erp_id.startswith(ERP_PREFIX): + errors.append( + f"erp_id must start with '{ERP_PREFIX}', got: {pkg.erp_id!r}" + ) + + # Rule 2: pipeline_version non-empty + if not pkg.pipeline_version.strip(): + errors.append("pipeline_version must not be empty") + + # Rule 3: example_version non-empty + if not pkg.example_version.strip(): + errors.append("example_version must not be empty") + + # Rule 4: creation_date ISO format + if not _ISO_DATE_RE.match(pkg.creation_date): + errors.append( + f"creation_date must be YYYY-MM-DD, got: {pkg.creation_date!r}" + ) + + # Rule 5: review_domain vocabulary + if pkg.review_domain not in VALID_REVIEW_DOMAINS: + errors.append( + f"review_domain must be one of {sorted(VALID_REVIEW_DOMAINS)}, " + f"got: {pkg.review_domain!r}" + ) + + # Rule 6: mock_candidates count + n = len(pkg.mock_candidates) + if n < MIN_MOCK_CANDIDATES or n > MAX_MOCK_CANDIDATES: + errors.append( + f"mock_candidates must have between {MIN_MOCK_CANDIDATES} and " + f"{MAX_MOCK_CANDIDATES} entries, got: {n}" + ) + + # Rule 7: validate each mock candidate + for i, cand in enumerate(pkg.mock_candidates): + errors.extend(_validate_mock_candidate(cand, i)) + + # Rule 8: at least one candidate must be included in the example + included = sum(1 for c in pkg.mock_candidates if c.include_in_example) + if included < 1: + errors.append( + "At least one mock candidate must have include_in_example=True" + ) + + # Rule 9: overall_clarity_rating Likert 1-5 + if pkg.overall_clarity_rating not in VALID_CLARITY_RATINGS: + errors.append( + f"overall_clarity_rating must be in {{1,2,3,4,5}}, " + f"got: {pkg.overall_clarity_rating}" + ) + + # Rule 10: synthesis_recommendation vocabulary + if pkg.synthesis_recommendation not in VALID_SYNTHESIS_RECOMMENDATIONS: + errors.append( + f"synthesis_recommendation must be one of " + f"{sorted(VALID_SYNTHESIS_RECOMMENDATIONS)}, " + f"got: {pkg.synthesis_recommendation!r}" + ) + + # Rule 11: reviewer_comments non-empty and length + if not pkg.reviewer_comments.strip(): + errors.append("reviewer_comments must not be empty") + elif len(pkg.reviewer_comments) > REVIEWER_COMMENTS_MAX_LENGTH: + errors.append( + f"reviewer_comments exceeds {REVIEWER_COMMENTS_MAX_LENGTH} chars " + f"(got {len(pkg.reviewer_comments)})" + ) + + # Rule 12: dry_lab_only must be True + if not pkg.dry_lab_only: + errors.append( + "dry_lab_only must be True — expert review examples use only " + "computational/dry-lab data" + ) + + # Rule 13: is_example_data must be True + if not pkg.is_example_data: + errors.append( + "is_example_data must be True — this schema is for example packages " + "only; use ReviewerQuestionnaire (RVQ-) for real review submissions" + ) + + # Rule 14: example_use_case non-empty + if not pkg.example_use_case.strip(): + errors.append("example_use_case must not be empty") + + # Rule 15: summary non-empty and length + if not pkg.summary.strip(): + errors.append("summary must not be empty") + elif len(pkg.summary) > SUMMARY_MAX_LENGTH: + errors.append( + f"summary exceeds {SUMMARY_MAX_LENGTH} chars " + f"(got {len(pkg.summary)})" + ) + + # Rule 16: notes length + if len(pkg.notes) > NOTES_MAX_LENGTH: + errors.append( + f"notes exceeds {NOTES_MAX_LENGTH} chars (got {len(pkg.notes)})" + ) + + # Warnings + if pkg.overall_clarity_rating <= 2: + errors.append( + f"WARNING: overall_clarity_rating ({pkg.overall_clarity_rating}) is low — " + "verify the example package is complete and well-structured" + ) + excluded_count = sum(1 for c in pkg.mock_candidates if not c.include_in_example) + if excluded_count > 0: + errors.append( + f"WARNING: {excluded_count} mock candidate(s) have " + "include_in_example=False — verify exclusion is intentional" + ) + if not pkg.notes.strip(): + errors.append( + "WARNING: notes is empty — consider documenting the example context" + ) + + return errors + + +def validate_dict(data: dict) -> List[str]: + """Validate a plain dict by constructing ExpertReviewExamplePackage first.""" + try: + cands_raw = data.get("mock_candidates", []) + cands = [MockCandidateSummary(**c) for c in cands_raw] + pkg_data = {k: v for k, v in data.items() if k != "mock_candidates"} + pkg = ExpertReviewExamplePackage(mock_candidates=cands, **pkg_data) + except TypeError as exc: + return [f"Schema construction error: {exc}"] + return validate(pkg) diff --git a/tests/evidence/test_expert_review_example_package.py b/tests/evidence/test_expert_review_example_package.py new file mode 100644 index 00000000..9cfec42b --- /dev/null +++ b/tests/evidence/test_expert_review_example_package.py @@ -0,0 +1,583 @@ +"""Tests for ExpertReviewExamplePackage schema — Phase E E10. + +Exactly 63 tests: valid baseline + each validation rule + edge cases + warnings. +""" + +from __future__ import annotations + +import pytest + +from openamp_foundry.evidence.expert_review_example_package import ( + ERP_PREFIX, + MOCK_CANDIDATE_ID_PREFIXES, + NOTES_MAX_LENGTH, + REVIEWER_COMMENTS_MAX_LENGTH, + SUMMARY_MAX_LENGTH, + VALID_REVIEW_DOMAINS, + VALID_SYNTHESIS_RECOMMENDATIONS, + ExpertReviewExamplePackage, + MockCandidateSummary, + validate, + validate_dict, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _mock_candidate( + candidate_id="MOCK-001", + sequence_length=20, + predicted_mic=2.0, + predicted_toxicity="low", + novelty_score=0.75, + include_in_example=True, +) -> MockCandidateSummary: + return MockCandidateSummary( + candidate_id=candidate_id, + sequence_length=sequence_length, + predicted_mic=predicted_mic, + predicted_toxicity=predicted_toxicity, + novelty_score=novelty_score, + include_in_example=include_in_example, + ) + + +def _valid_pkg(**overrides) -> ExpertReviewExamplePackage: + defaults = dict( + erp_id="ERP-20240315-001", + pipeline_version="v2.1.0", + example_version="v1.0", + creation_date="2024-03-15", + review_domain="antimicrobial_activity", + mock_candidates=[ + _mock_candidate("MOCK-001"), + _mock_candidate("MOCK-002", predicted_mic=4.0, novelty_score=0.6), + ], + overall_clarity_rating=4, + synthesis_recommendation="proceed_with_conditions", + reviewer_comments=( + "Toy candidate MOCK-001 shows MIC 2 µg/mL and low toxicity. " + "This example demonstrates the expected review format." + ), + dry_lab_only=True, + is_example_data=True, + example_use_case=( + "Demonstrates the format for antimicrobial activity review " + "with two toy candidates of varying MIC profiles." + ), + summary=( + "Example review package: 2 mock candidates, " + "one recommended for conditional synthesis." + ), + notes="", + ) + defaults.update(overrides) + return ExpertReviewExamplePackage(**defaults) + + +def _valid() -> ExpertReviewExamplePackage: + return _valid_pkg() + + +def _errors(p): + return [e for e in validate(p) if not e.startswith("WARNING:")] + + +def _warns(p): + return [e for e in validate(p) if e.startswith("WARNING:")] + + +# --------------------------------------------------------------------------- +# Group 1: Valid baseline (3 tests) +# --------------------------------------------------------------------------- + +class TestValidBaseline: + def test_valid_returns_no_errors(self): + assert _errors(_valid()) == [] + + def test_valid_with_notes(self): + p = _valid_pkg(notes="Used for onboarding new reviewers.") + assert _errors(p) == [] + + def test_valid_single_candidate(self): + p = _valid_pkg( + mock_candidates=[_mock_candidate("TOY-001")], + summary="Single-candidate example.", + ) + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 2: Rule 1 — erp_id prefix (4 tests) +# --------------------------------------------------------------------------- + +class TestErpIdPrefix: + def test_wrong_prefix_rejected(self): + p = _valid_pkg(erp_id="RVQ-001") + assert any("erp_id" in e for e in _errors(p)) + + def test_lowercase_prefix_rejected(self): + p = _valid_pkg(erp_id="erp-001") + assert any("erp_id" in e for e in _errors(p)) + + def test_no_prefix_rejected(self): + p = _valid_pkg(erp_id="001") + assert any("erp_id" in e for e in _errors(p)) + + def test_correct_prefix_accepted(self): + p = _valid_pkg(erp_id="ERP-2024-001") + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 3: Rule 2+3 — pipeline_version and example_version (4 tests) +# --------------------------------------------------------------------------- + +class TestVersionFields: + def test_empty_pipeline_version_rejected(self): + p = _valid_pkg(pipeline_version="") + assert any("pipeline_version" in e for e in _errors(p)) + + def test_whitespace_pipeline_version_rejected(self): + p = _valid_pkg(pipeline_version=" ") + assert any("pipeline_version" in e for e in _errors(p)) + + def test_empty_example_version_rejected(self): + p = _valid_pkg(example_version="") + assert any("example_version" in e for e in _errors(p)) + + def test_valid_versions(self): + p = _valid_pkg(pipeline_version="v3.0.0", example_version="v2.1") + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 4: Rule 4 — creation_date ISO format (3 tests) +# --------------------------------------------------------------------------- + +class TestCreationDate: + def test_invalid_format_rejected(self): + p = _valid_pkg(creation_date="March 15, 2024") + assert any("creation_date" in e for e in _errors(p)) + + def test_wrong_separator_rejected(self): + p = _valid_pkg(creation_date="2024/03/15") + assert any("creation_date" in e for e in _errors(p)) + + def test_valid_iso_date(self): + p = _valid_pkg(creation_date="2025-01-01") + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 5: Rule 5 — review_domain vocabulary (4 tests) +# --------------------------------------------------------------------------- + +class TestReviewDomain: + def test_invalid_domain_rejected(self): + p = _valid_pkg(review_domain="random_domain") + assert any("review_domain" in e for e in _errors(p)) + + def test_empty_rejected(self): + p = _valid_pkg(review_domain="") + assert any("review_domain" in e for e in _errors(p)) + + @pytest.mark.parametrize("domain", sorted(VALID_REVIEW_DOMAINS)) + def test_all_valid_domains_accepted(self, domain): + p = _valid_pkg(review_domain=domain) + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 6: Rule 6 — mock_candidates count (4 tests) +# --------------------------------------------------------------------------- + +class TestMockCandidatesCount: + def test_empty_list_rejected(self): + p = _valid_pkg(mock_candidates=[]) + assert any("mock_candidates" in e for e in _errors(p)) + + def test_too_many_rejected(self): + p = _valid_pkg( + mock_candidates=[_mock_candidate(f"MOCK-{i:03d}") for i in range(11)] + ) + assert any("mock_candidates" in e for e in _errors(p)) + + def test_exactly_ten_accepted(self): + p = _valid_pkg( + mock_candidates=[_mock_candidate(f"MOCK-{i:03d}") for i in range(10)] + ) + assert _errors(p) == [] + + def test_one_candidate_accepted(self): + p = _valid_pkg( + mock_candidates=[_mock_candidate("DEMO-001")], + summary="Single demo candidate.", + ) + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 7: Rule 7 — mock candidate ID prefixes (4 tests) +# --------------------------------------------------------------------------- + +class TestMockCandidateId: + def test_real_id_prefix_rejected(self): + p = _valid_pkg( + mock_candidates=[_mock_candidate("PEP-001")] + ) + assert any("candidate_id" in e for e in _errors(p)) + + @pytest.mark.parametrize("prefix", sorted(MOCK_CANDIDATE_ID_PREFIXES)) + def test_all_mock_prefixes_accepted(self, prefix): + p = _valid_pkg( + mock_candidates=[_mock_candidate(f"{prefix}001")] + ) + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 8: Rule 7 — mock candidate field validation (4 tests) +# --------------------------------------------------------------------------- + +class TestMockCandidateFields: + def test_zero_sequence_length_rejected(self): + p = _valid_pkg( + mock_candidates=[_mock_candidate("MOCK-001", sequence_length=0)] + ) + assert any("sequence_length" in e for e in _errors(p)) + + def test_zero_predicted_mic_rejected(self): + p = _valid_pkg( + mock_candidates=[_mock_candidate("MOCK-001", predicted_mic=0.0)] + ) + assert any("predicted_mic" in e for e in _errors(p)) + + def test_invalid_toxicity_rejected(self): + p = _valid_pkg( + mock_candidates=[_mock_candidate("MOCK-001", predicted_toxicity="unknown")] + ) + assert any("predicted_toxicity" in e for e in _errors(p)) + + def test_novelty_out_of_range_rejected(self): + p = _valid_pkg( + mock_candidates=[_mock_candidate("MOCK-001", novelty_score=1.5)] + ) + assert any("novelty_score" in e for e in _errors(p)) + + +# --------------------------------------------------------------------------- +# Group 9: Rule 8 — at least one included candidate (3 tests) +# --------------------------------------------------------------------------- + +class TestIncludeInExample: + def test_all_excluded_rejected(self): + p = _valid_pkg( + mock_candidates=[ + _mock_candidate("MOCK-001", include_in_example=False), + _mock_candidate("MOCK-002", include_in_example=False), + ] + ) + assert any("include_in_example" in e for e in _errors(p)) + + def test_one_included_accepted(self): + p = _valid_pkg( + mock_candidates=[ + _mock_candidate("MOCK-001", include_in_example=True), + _mock_candidate("MOCK-002", include_in_example=False), + ] + ) + assert _errors(p) == [] + + def test_all_included_accepted(self): + p = _valid_pkg( + mock_candidates=[ + _mock_candidate("MOCK-001", include_in_example=True), + _mock_candidate("MOCK-002", include_in_example=True), + ] + ) + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 10: Rule 9 — overall_clarity_rating Likert (3 tests) +# --------------------------------------------------------------------------- + +class TestClarityRating: + def test_zero_rejected(self): + p = _valid_pkg(overall_clarity_rating=0) + assert any("overall_clarity_rating" in e for e in _errors(p)) + + def test_six_rejected(self): + p = _valid_pkg(overall_clarity_rating=6) + assert any("overall_clarity_rating" in e for e in _errors(p)) + + def test_one_through_five_accepted(self): + for rating in range(1, 6): + p = _valid_pkg(overall_clarity_rating=rating) + assert _errors(p) == [], f"Rating {rating} should be valid" + + +# --------------------------------------------------------------------------- +# Group 11: Rule 10 — synthesis_recommendation vocabulary (4 tests) +# --------------------------------------------------------------------------- + +class TestSynthesisRecommendation: + def test_invalid_recommendation_rejected(self): + p = _valid_pkg(synthesis_recommendation="maybe") + assert any("synthesis_recommendation" in e for e in _errors(p)) + + def test_empty_rejected(self): + p = _valid_pkg(synthesis_recommendation="") + assert any("synthesis_recommendation" in e for e in _errors(p)) + + @pytest.mark.parametrize("rec", sorted(VALID_SYNTHESIS_RECOMMENDATIONS)) + def test_all_valid_recommendations_accepted(self, rec): + p = _valid_pkg(synthesis_recommendation=rec) + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 12: Rule 11 — reviewer_comments (3 tests) +# --------------------------------------------------------------------------- + +class TestReviewerComments: + def test_empty_rejected(self): + p = _valid_pkg(reviewer_comments="") + assert any("reviewer_comments" in e for e in _errors(p)) + + def test_too_long_rejected(self): + p = _valid_pkg(reviewer_comments="x" * (REVIEWER_COMMENTS_MAX_LENGTH + 1)) + assert any("reviewer_comments" in e for e in _errors(p)) + + def test_at_limit_accepted(self): + p = _valid_pkg(reviewer_comments="x" * REVIEWER_COMMENTS_MAX_LENGTH) + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 13: Rule 12+13 — dry_lab_only and is_example_data (3 tests) +# --------------------------------------------------------------------------- + +class TestSafetyFlags: + def test_dry_lab_only_false_rejected(self): + p = _valid_pkg(dry_lab_only=False) + assert any("dry_lab_only" in e for e in _errors(p)) + + def test_is_example_data_false_rejected(self): + p = _valid_pkg(is_example_data=False) + assert any("is_example_data" in e for e in _errors(p)) + + def test_both_true_accepted(self): + p = _valid_pkg(dry_lab_only=True, is_example_data=True) + assert _errors(p) == [] + + +# --------------------------------------------------------------------------- +# Group 14: Rule 14+15+16 — example_use_case, summary, notes (4 tests) +# --------------------------------------------------------------------------- + +class TestTextFields: + def test_empty_example_use_case_rejected(self): + p = _valid_pkg(example_use_case="") + assert any("example_use_case" in e for e in _errors(p)) + + def test_empty_summary_rejected(self): + p = _valid_pkg(summary="") + assert any("summary" in e for e in _errors(p)) + + def test_summary_too_long_rejected(self): + p = _valid_pkg(summary="x" * (SUMMARY_MAX_LENGTH + 1)) + assert any("summary" in e for e in _errors(p)) + + def test_notes_too_long_rejected(self): + p = _valid_pkg(notes="n" * (NOTES_MAX_LENGTH + 1)) + assert any("notes" in e for e in _errors(p)) + + +# --------------------------------------------------------------------------- +# Group 15: Warnings (6 tests) +# --------------------------------------------------------------------------- + +class TestWarnings: + def test_low_clarity_rating_triggers_warning(self): + p = _valid_pkg(overall_clarity_rating=2) + warns = _warns(p) + assert any("overall_clarity_rating" in w for w in warns) + + def test_excluded_candidates_triggers_warning(self): + p = _valid_pkg( + mock_candidates=[ + _mock_candidate("MOCK-001", include_in_example=True), + _mock_candidate("MOCK-002", include_in_example=False), + ] + ) + warns = _warns(p) + assert any("include_in_example=False" in w for w in warns) + + def test_empty_notes_triggers_warning(self): + p = _valid_pkg(notes="") + warns = _warns(p) + assert any("notes" in w.lower() for w in warns) + + def test_notes_present_suppresses_notes_warning(self): + p = _valid_pkg(notes="Used for partner onboarding.") + warns = _warns(p) + assert not any("notes is empty" in w for w in warns) + + def test_high_clarity_rating_no_warning(self): + p = _valid_pkg(overall_clarity_rating=5) + warns = _warns(p) + assert not any("overall_clarity_rating" in w for w in warns) + + def test_all_included_no_exclusion_warning(self): + p = _valid_pkg( + mock_candidates=[ + _mock_candidate("MOCK-001", include_in_example=True), + _mock_candidate("MOCK-002", include_in_example=True), + ] + ) + warns = _warns(p) + assert not any("include_in_example=False" in w for w in warns) + + +# --------------------------------------------------------------------------- +# Group 16: validate_dict (4 tests) +# --------------------------------------------------------------------------- + +class TestValidateDict: + def test_valid_dict_returns_no_errors(self): + data = dict( + erp_id="ERP-20240315-001", + pipeline_version="v2.1.0", + example_version="v1.0", + creation_date="2024-03-15", + review_domain="antimicrobial_activity", + mock_candidates=[ + dict( + candidate_id="MOCK-001", + sequence_length=20, + predicted_mic=2.0, + predicted_toxicity="low", + novelty_score=0.75, + include_in_example=True, + ) + ], + overall_clarity_rating=4, + synthesis_recommendation="proceed", + reviewer_comments="This is a toy example for reviewer onboarding.", + dry_lab_only=True, + is_example_data=True, + example_use_case="Demonstrates antimicrobial activity review.", + summary="One mock candidate, proceed recommendation.", + ) + errs = [e for e in validate_dict(data) if not e.startswith("WARNING:")] + assert errs == [] + + def test_missing_required_field_returns_error(self): + data = dict(erp_id="ERP-001", pipeline_version="v1.0") + result = validate_dict(data) + assert any("Schema construction error" in e for e in result) + + def test_invalid_field_caught_by_dict_validator(self): + data = dict( + erp_id="WRONG-001", + pipeline_version="v1.0", + example_version="v1.0", + creation_date="2024-01-01", + review_domain="antimicrobial_activity", + mock_candidates=[ + dict( + candidate_id="MOCK-001", + sequence_length=20, + predicted_mic=2.0, + predicted_toxicity="low", + novelty_score=0.5, + include_in_example=True, + ) + ], + overall_clarity_rating=3, + synthesis_recommendation="proceed", + reviewer_comments="Test.", + dry_lab_only=True, + is_example_data=True, + example_use_case="Test example.", + summary="Test.", + ) + result = validate_dict(data) + assert any("erp_id" in e for e in result) + + def test_extra_notes_in_dict(self): + data = dict( + erp_id="ERP-20240315-002", + pipeline_version="v2.1.0", + example_version="v1.0", + creation_date="2024-03-15", + review_domain="toxicity_safety", + mock_candidates=[ + dict( + candidate_id="TOY-001", + sequence_length=15, + predicted_mic=8.0, + predicted_toxicity="moderate", + novelty_score=0.4, + include_in_example=True, + ) + ], + overall_clarity_rating=5, + synthesis_recommendation="reject", + reviewer_comments="TOY-001 shows moderate toxicity; reject for synthesis.", + dry_lab_only=True, + is_example_data=True, + example_use_case="Demonstrates rejection due to toxicity concerns.", + summary="One toy candidate rejected for safety.", + notes="Part of standard onboarding examples.", + ) + errs = [e for e in validate_dict(data) if not e.startswith("WARNING:")] + assert errs == [] + + +# --------------------------------------------------------------------------- +# Group 17: Edge cases (7 tests) +# --------------------------------------------------------------------------- + +class TestEdgeCases: + def test_erp_prefix_constant(self): + assert ERP_PREFIX == "ERP-" + + def test_all_valid_review_domains(self): + for domain in sorted(VALID_REVIEW_DOMAINS): + p = _valid_pkg(review_domain=domain) + assert _errors(p) == [], f"Domain {domain} should be valid" + + def test_all_valid_recommendations(self): + for rec in sorted(VALID_SYNTHESIS_RECOMMENDATIONS): + p = _valid_pkg(synthesis_recommendation=rec) + assert _errors(p) == [], f"Recommendation {rec} should be valid" + + def test_all_mock_prefixes_accepted(self): + for prefix in sorted(MOCK_CANDIDATE_ID_PREFIXES): + p = _valid_pkg( + mock_candidates=[_mock_candidate(f"{prefix}001")] + ) + assert _errors(p) == [], f"Prefix {prefix} should be valid" + + def test_max_candidates_accepted(self): + p = _valid_pkg( + mock_candidates=[ + _mock_candidate(f"MOCK-{i:03d}") for i in range(10) + ] + ) + assert _errors(p) == [] + + def test_summary_at_limit_accepted(self): + p = _valid_pkg(summary="s" * SUMMARY_MAX_LENGTH) + assert _errors(p) == [] + + def test_negative_mic_rejected(self): + p = _valid_pkg( + mock_candidates=[_mock_candidate("MOCK-001", predicted_mic=-1.0)] + ) + assert any("predicted_mic" in e for e in _errors(p)) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index b576f79b..18c1e481 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 6107 +BASELINE = 6170 TOLERANCE = 0.05 # allow 5% fluctuation