From b73a39af0694f1b8390dd6850fd2ae0bb83c84c3 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 09:57:35 +0700 Subject: [PATCH] feat: Phase B B7 candidate rejection certificate (CRC-) -- failures become first-class auditable artifacts --- .../CANDIDATE_REJECTION_CERTIFICATE_GUIDE.md | 51 +++ docs/evidence/METRICS_CURRENT.md | 9 + docs/research/NEXT_100_PR_MAP.md | 2 +- .../candidate_rejection_certificate.py | 194 ++++++++ .../test_candidate_rejection_certificate.py | 431 ++++++++++++++++++ tests/test_test_count_regression.py | 2 +- 6 files changed, 687 insertions(+), 2 deletions(-) create mode 100644 docs/evidence/CANDIDATE_REJECTION_CERTIFICATE_GUIDE.md create mode 100644 src/openamp_foundry/evidence/candidate_rejection_certificate.py create mode 100644 tests/evidence/test_candidate_rejection_certificate.py diff --git a/docs/evidence/CANDIDATE_REJECTION_CERTIFICATE_GUIDE.md b/docs/evidence/CANDIDATE_REJECTION_CERTIFICATE_GUIDE.md new file mode 100644 index 00000000..8eeff3da --- /dev/null +++ b/docs/evidence/CANDIDATE_REJECTION_CERTIFICATE_GUIDE.md @@ -0,0 +1,51 @@ +# Candidate Rejection Certificate Guide (CRC-) + +## Purpose + +A `CandidateRejectionCertificate` makes pipeline rejections first-class auditable artifacts. +When any gate rejects a candidate, a CRC- record documents: which gate, which reason, the evidence at time of rejection, and the proof-ladder level reached before rejection. + +This closes the feedback loop: failures are as durable as passes. + +## Required Fields + +| Field | Type | Description | +|-------|------|-------------| +| `crc_id` | str | Unique ID starting with `CRC-` | +| `pipeline_version` | str | Pipeline version at rejection time | +| `candidate_id` | str | ID of the rejected candidate | +| `sequence` | str | Amino acid sequence | +| `rejection_date` | str | ISO 8601 date | +| `rejection_gate` | str | Gate that triggered rejection (controlled vocab) | +| `rejection_reason` | str | Reason code (controlled vocab) | +| `evidence_summary` | str | Human-readable evidence at rejection | +| `proof_ladder_level_at_rejection` | str | Highest level reached before rejection | +| `dry_lab_only` | bool | Must be True | + +## Optional Fields + +| Field | Default | Description | +|-------|---------|-------------| +| `scores` | {} | Score dict at rejection time | +| `notes` | "" | Reviewer notes (max 400 chars) | + +## Validation Rules + +1. `crc_id` must start with `CRC-` +2. `pipeline_version` non-empty +3. `candidate_id` non-empty +4. `sequence` non-empty, standard amino acids only +5. `rejection_date` ISO 8601 format +6. `rejection_gate` in `VALID_REJECTION_GATES` +7. `rejection_reason` in `VALID_REJECTION_REASONS` +8. `evidence_summary` non-empty +9. `proof_ladder_level_at_rejection` in `VALID_PROOF_LADDER_LEVELS` +10. `dry_lab_only` must be True +11. `proof_ladder_level_at_rejection` capped at `multi_signal_candidate_evidence` when `dry_lab_only=True` +12. `scores` must be a dict +13. `notes` ≤400 characters + +## Claim Safety + +CRC- records are dry-lab artifacts. `dry_lab_only=True` is enforced. +The proof ladder is capped at `multi_signal_candidate_evidence` (level 4). diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 3745006a..884b659e 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -9,6 +9,15 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak ## Changelog +### v0.10.31 — Phase B B7: Candidate rejection certificate (CRC-) +- Added `src/openamp_foundry/evidence/candidate_rejection_certificate.py` +- CRC- schema: 12 fields (crc_id, pipeline_version, candidate_id, sequence, rejection_date, rejection_gate, rejection_reason, evidence_summary, proof_ladder_level_at_rejection, dry_lab_only, scores, notes) +- 13 validation rules: CRC- prefix, sequence validation, ISO date, gate/reason controlled vocab (10 gates, 12 reasons), evidence_summary non-empty, dry_lab_only=True enforced, proof-ladder cap at multi_signal_candidate_evidence +- 2 warnings: short evidence_summary, empty notes +- 63 tests in `tests/evidence/test_candidate_rejection_certificate.py` +- BASELINE 6520→6583 +- Closes Phase B B7 — pipeline rejections are now first-class auditable artifacts + ### v0.10.30 — Phase B B6: Human-readable certificate report - Added `src/openamp_foundry/evidence/certificate_report.py` — build_certificate_report() function - Converts cert dict to formatted text with labelled sections: CANDIDATE, PROOF LADDER, SCORES (with not-biological-proof notice), CHEAP-EXPLANATION CHECK, SELECTION REASON, KNOWN FAILURE MODES, RECOMMENDED NEXT STEPS, REFERENCES CHECKED, optional QUALITY TIER diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index d6b4d849..d4de904b 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -47,7 +47,7 @@ Make candidate evidence packages impossible to confuse with biological proof. | B4 | Add `release_status` field. | Supports staged release. | C/D | | B5 | Add certificate quality-tier validator (complete). | assess_certificate_quality() computes draft/internal_review/external_review_ready tier; missing-field detection, forbidden-claim violation check, external-review gate; 63 tests in tests/test_certificate_quality_validator.py. | B | | B6 | Add human-readable certificate report (complete). | build_certificate_report() in certificate_report.py converts cert dict to formatted text with sections: identity, proof ladder, scores, cheap-explanation check, selection reason, failure modes, next steps, references, optional quality tier; 63 tests in tests/test_certificate_report.py. | B | -| B7 | Add candidate rejection certificate support. | Failures become artifacts. | B | +| B7 | Add candidate rejection certificate support (complete). | CRC- schema: 12 fields, 13 rules, VALID_REJECTION_GATES (10 values), VALID_REJECTION_REASONS (12 values), dry_lab_only enforced, proof-ladder cap at multi_signal; makes pipeline rejections first-class auditable artifacts; 63 tests. | B | | B8 | Link certificates to run manifest hashes. | Stronger reproducibility. | B | | B9 | Add test that dry-lab certificates cannot include forbidden claims (complete). | CI gate: tests/test_certificate_claim_discipline.py scans all cert text fields against RISKY_PATTERNS and FORBIDDEN_CLAIM_PATTERNS; default cert proven clean; forbidden phrases confirmed detectable. 35 tests total. | B/C | | B10 | Add external-review packet generator that bundles certificates. | Better partner review. | C/D | diff --git a/src/openamp_foundry/evidence/candidate_rejection_certificate.py b/src/openamp_foundry/evidence/candidate_rejection_certificate.py new file mode 100644 index 00000000..b9c61bd7 --- /dev/null +++ b/src/openamp_foundry/evidence/candidate_rejection_certificate.py @@ -0,0 +1,194 @@ +"""CandidateRejectionCertificate — Phase B B7. + +When a candidate fails any gate, this schema captures the rejection as a +first-class auditable artifact. Makes failures as durable as passes. + +ID prefix: CRC- +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List + +CRC_PREFIX = "CRC-" + +VALID_REJECTION_GATES = frozenset({ + "wave0_5_safety_gate", + "novelty_gate", + "activity_threshold", + "diversity_filter", + "hemolysis_gate", + "toxicity_gate", + "synthesis_feasibility_gate", + "manual_review", + "claim_boundary_gate", + "calibration_gate", +}) + +VALID_REJECTION_REASONS = frozenset({ + "safety_score_below_threshold", + "activity_score_below_threshold", + "insufficient_novelty", + "hemolysis_risk_too_high", + "toxicity_risk_too_high", + "synthesis_infeasible", + "duplicate_of_known", + "diversity_cluster_full", + "human_reviewer_rejected", + "claim_boundary_violation", + "calibration_refused", + "overclaiming_language_detected", +}) + +VALID_PROOF_LADDER_LEVELS = ( + "valid_input", + "reproducible_dry_lab_features", + "baseline_triaged", + "leakage_aware_benchmark", + "multi_signal_candidate_evidence", + "expert_reviewed_assay_proposal", + "initial_qualified_assay_result", + "safety_adjusted_follow_up_signal", + "independent_replication", + "reusable_discovery_loop", +) + +_ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(T[\d:.+Z-]+)?$") + + +@dataclass +class CandidateRejectionCertificate: + crc_id: str + pipeline_version: str + candidate_id: str + sequence: str + rejection_date: str + rejection_gate: str + rejection_reason: str + evidence_summary: str + proof_ladder_level_at_rejection: str + dry_lab_only: bool + scores: dict = field(default_factory=dict) + notes: str = "" + + def validate(self) -> list[str]: + errors: list[str] = [] + warnings: list[str] = [] + + # Rule 1: CRC- prefix + if not self.crc_id.startswith(CRC_PREFIX): + errors.append(f"crc_id must start with '{CRC_PREFIX}', got {self.crc_id!r}") + + # Rule 2: pipeline_version non-empty + if not self.pipeline_version.strip(): + errors.append("pipeline_version must not be empty") + + # Rule 3: candidate_id non-empty + if not self.candidate_id.strip(): + errors.append("candidate_id must not be empty") + + # Rule 4: sequence non-empty and uppercase amino acids + if not self.sequence.strip(): + errors.append("sequence must not be empty") + elif not re.match(r"^[ACDEFGHIKLMNPQRSTVWY]+$", self.sequence.upper()): + errors.append( + "sequence must contain only standard amino acid characters" + ) + + # Rule 5: rejection_date ISO format + if not _ISO_DATE_RE.match(self.rejection_date): + errors.append( + f"rejection_date must be ISO 8601 date, got {self.rejection_date!r}" + ) + + # Rule 6: rejection_gate in controlled vocab + if self.rejection_gate not in VALID_REJECTION_GATES: + errors.append( + f"rejection_gate must be one of {sorted(VALID_REJECTION_GATES)}, " + f"got {self.rejection_gate!r}" + ) + + # Rule 7: rejection_reason in controlled vocab + if self.rejection_reason not in VALID_REJECTION_REASONS: + errors.append( + f"rejection_reason must be one of {sorted(VALID_REJECTION_REASONS)}, " + f"got {self.rejection_reason!r}" + ) + + # Rule 8: evidence_summary non-empty + if not self.evidence_summary.strip(): + errors.append("evidence_summary must not be empty") + + # Rule 9: proof_ladder_level_at_rejection in controlled vocab + if self.proof_ladder_level_at_rejection not in VALID_PROOF_LADDER_LEVELS: + errors.append( + f"proof_ladder_level_at_rejection must be one of " + f"{VALID_PROOF_LADDER_LEVELS}, got {self.proof_ladder_level_at_rejection!r}" + ) + + # Rule 10: dry_lab_only must be True + if not self.dry_lab_only: + errors.append( + "dry_lab_only must be True — rejection certificates are dry-lab artifacts" + ) + + # Rule 11: proof_ladder_level_at_rejection <= multi_signal_candidate_evidence if dry_lab_only + dry_lab_cap_index = VALID_PROOF_LADDER_LEVELS.index("multi_signal_candidate_evidence") + if self.proof_ladder_level_at_rejection in VALID_PROOF_LADDER_LEVELS: + level_index = VALID_PROOF_LADDER_LEVELS.index(self.proof_ladder_level_at_rejection) + if self.dry_lab_only and level_index > dry_lab_cap_index: + errors.append( + f"dry_lab_only=True caps proof_ladder_level_at_rejection at " + f"'multi_signal_candidate_evidence' (level 4), " + f"got {self.proof_ladder_level_at_rejection!r} (level {level_index})" + ) + + # Rule 12: scores must be dict if provided + if not isinstance(self.scores, dict): + errors.append("scores must be a dict") + + # Rule 13: notes length cap + if len(self.notes) > 400: + errors.append(f"notes must be ≤400 characters, got {len(self.notes)}") + + # Warning: evidence_summary short + if len(self.evidence_summary.strip()) < 20: + warnings.append( + "evidence_summary is very short; provide enough context to " + "reconstruct the rejection decision" + ) + + # Warning: notes empty + if not self.notes.strip(): + warnings.append( + "notes is empty; consider adding reviewer context or " + "follow-up instructions" + ) + + if errors: + return errors + return [f"WARNING: {w}" for w in warnings] if warnings else [] + + +def validate_dict(data: dict) -> list[str]: + """Validate a dict representation of CandidateRejectionCertificate.""" + try: + record = CandidateRejectionCertificate( + crc_id=data.get("crc_id", ""), + pipeline_version=data.get("pipeline_version", ""), + candidate_id=data.get("candidate_id", ""), + sequence=data.get("sequence", ""), + rejection_date=data.get("rejection_date", ""), + rejection_gate=data.get("rejection_gate", ""), + rejection_reason=data.get("rejection_reason", ""), + evidence_summary=data.get("evidence_summary", ""), + proof_ladder_level_at_rejection=data.get("proof_ladder_level_at_rejection", ""), + dry_lab_only=data.get("dry_lab_only", False), + scores=data.get("scores", {}), + notes=data.get("notes", ""), + ) + return record.validate() + except Exception as exc: + return [f"Construction error: {exc}"] diff --git a/tests/evidence/test_candidate_rejection_certificate.py b/tests/evidence/test_candidate_rejection_certificate.py new file mode 100644 index 00000000..0da13236 --- /dev/null +++ b/tests/evidence/test_candidate_rejection_certificate.py @@ -0,0 +1,431 @@ +"""Tests for CandidateRejectionCertificate — Phase B B7. + +63 tests across 7 groups. +""" + +from __future__ import annotations + +import pytest + +from openamp_foundry.evidence.candidate_rejection_certificate import ( + CRC_PREFIX, + VALID_PROOF_LADDER_LEVELS, + VALID_REJECTION_GATES, + VALID_REJECTION_REASONS, + CandidateRejectionCertificate, + validate_dict, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _valid() -> CandidateRejectionCertificate: + return CandidateRejectionCertificate( + crc_id="CRC-0001", + pipeline_version="0.10.30", + candidate_id="AMPF-001", + sequence="KWKLFKKIGAVLKVL", + rejection_date="2024-06-01", + rejection_gate="activity_threshold", + rejection_reason="activity_score_below_threshold", + evidence_summary="Activity score 0.31 below threshold 0.60; candidate rejected by activity gate.", + proof_ladder_level_at_rejection="baseline_triaged", + dry_lab_only=True, + scores={"activity": 0.31, "safety": 0.85}, + notes="Reviewed by pipeline_owner.", + ) + + +def _valid_dict() -> dict: + r = _valid() + return { + "crc_id": r.crc_id, + "pipeline_version": r.pipeline_version, + "candidate_id": r.candidate_id, + "sequence": r.sequence, + "rejection_date": r.rejection_date, + "rejection_gate": r.rejection_gate, + "rejection_reason": r.rejection_reason, + "evidence_summary": r.evidence_summary, + "proof_ladder_level_at_rejection": r.proof_ladder_level_at_rejection, + "dry_lab_only": r.dry_lab_only, + "scores": r.scores, + "notes": r.notes, + } + + +# --------------------------------------------------------------------------- +# Group 1: ValidRecord (8 tests) +# --------------------------------------------------------------------------- + +class TestValidRecord: + def test_valid_record_passes(self): + assert _valid().validate() == [] + + def test_valid_dict_passes(self): + assert validate_dict(_valid_dict()) == [] + + def test_crc_prefix_constant(self): + assert CRC_PREFIX == "CRC-" + + def test_valid_rejection_gates_non_empty(self): + assert len(VALID_REJECTION_GATES) >= 8 + + def test_valid_rejection_reasons_non_empty(self): + assert len(VALID_REJECTION_REASONS) >= 8 + + def test_valid_proof_ladder_levels_count(self): + assert len(VALID_PROOF_LADDER_LEVELS) == 10 + + def test_record_with_warnings_only_passes(self): + r = _valid() + r.notes = "" + result = r.validate() + # warnings only, no errors + assert all("WARNING" in msg for msg in result) + + def test_record_with_notes_and_evidence_no_warnings(self): + r = _valid() + r.notes = "Reviewed." + result = r.validate() + assert result == [] + + +# --------------------------------------------------------------------------- +# Group 2: IDAndVersionRules (10 tests) +# --------------------------------------------------------------------------- + +class TestIDAndVersionRules: + def test_missing_crc_prefix_fails(self): + r = _valid() + r.crc_id = "WRONG-001" + assert any("CRC-" in e for e in r.validate()) + + def test_empty_crc_id_fails(self): + r = _valid() + r.crc_id = "" + errors = r.validate() + assert any("crc_id" in e for e in errors) + + def test_empty_pipeline_version_fails(self): + r = _valid() + r.pipeline_version = "" + assert any("pipeline_version" in e for e in r.validate()) + + def test_whitespace_pipeline_version_fails(self): + r = _valid() + r.pipeline_version = " " + assert any("pipeline_version" in e for e in r.validate()) + + def test_empty_candidate_id_fails(self): + r = _valid() + r.candidate_id = "" + assert any("candidate_id" in e for e in r.validate()) + + def test_crc_prefix_with_suffix_passes(self): + r = _valid() + r.crc_id = "CRC-9999" + assert r.validate() == [] + + def test_lowercase_crc_fails(self): + r = _valid() + r.crc_id = "crc-001" + errors = r.validate() + assert any("CRC-" in e for e in errors) + + def test_pipeline_version_with_dots_passes(self): + r = _valid() + r.pipeline_version = "1.2.3" + assert r.validate() == [] + + def test_long_candidate_id_passes(self): + r = _valid() + r.candidate_id = "VERY-LONG-CANDIDATE-ID-0000001" + assert r.validate() == [] + + def test_validate_dict_wrong_prefix(self): + d = _valid_dict() + d["crc_id"] = "WRONG-001" + errors = validate_dict(d) + assert any("CRC-" in e for e in errors) + + +# --------------------------------------------------------------------------- +# Group 3: SequenceAndDateRules (8 tests) +# --------------------------------------------------------------------------- + +class TestSequenceAndDateRules: + def test_empty_sequence_fails(self): + r = _valid() + r.sequence = "" + assert any("sequence" in e for e in r.validate()) + + def test_invalid_sequence_chars_fails(self): + r = _valid() + r.sequence = "KWKLFKK123" + assert any("amino acid" in e for e in r.validate()) + + def test_valid_sequence_passes(self): + r = _valid() + r.sequence = "ACDEFGHIKLMNPQRSTVWY" + assert r.validate() == [] + + def test_non_iso_date_fails(self): + r = _valid() + r.rejection_date = "June 1 2024" + assert any("ISO 8601" in e for e in r.validate()) + + def test_valid_date_passes(self): + r = _valid() + r.rejection_date = "2024-12-31" + assert r.validate() == [] + + def test_datetime_format_passes(self): + r = _valid() + r.rejection_date = "2024-06-01T12:00:00Z" + assert r.validate() == [] + + def test_empty_date_fails(self): + r = _valid() + r.rejection_date = "" + assert any("rejection_date" in e for e in r.validate()) + + def test_lowercase_sequence_validated_by_content(self): + r = _valid() + r.sequence = "kwklfkkigavlkvl" + # lowercase but valid AA chars when uppercased + assert r.validate() == [] + + +# --------------------------------------------------------------------------- +# Group 4: GateAndReasonRules (10 tests) +# --------------------------------------------------------------------------- + +class TestGateAndReasonRules: + def test_invalid_gate_fails(self): + r = _valid() + r.rejection_gate = "unknown_gate" + assert any("rejection_gate" in e for e in r.validate()) + + def test_all_valid_gates_pass(self): + for gate in VALID_REJECTION_GATES: + r = _valid() + r.rejection_gate = gate + # Pick a compatible reason + errors = [e for e in r.validate() if "rejection_gate" in e] + assert errors == [], f"Gate {gate!r} should be valid" + + def test_invalid_reason_fails(self): + r = _valid() + r.rejection_reason = "not_a_valid_reason" + assert any("rejection_reason" in e for e in r.validate()) + + def test_all_valid_reasons_pass(self): + for reason in VALID_REJECTION_REASONS: + r = _valid() + r.rejection_reason = reason + errors = [e for e in r.validate() if "rejection_reason" in e] + assert errors == [], f"Reason {reason!r} should be valid" + + def test_empty_gate_fails(self): + r = _valid() + r.rejection_gate = "" + assert any("rejection_gate" in e for e in r.validate()) + + def test_empty_reason_fails(self): + r = _valid() + r.rejection_reason = "" + assert any("rejection_reason" in e for e in r.validate()) + + def test_wave0_5_safety_gate_passes(self): + r = _valid() + r.rejection_gate = "wave0_5_safety_gate" + errors = [e for e in r.validate() if "rejection_gate" in e] + assert errors == [] + + def test_hemolysis_gate_passes(self): + r = _valid() + r.rejection_gate = "hemolysis_gate" + errors = [e for e in r.validate() if "rejection_gate" in e] + assert errors == [] + + def test_manual_review_gate_passes(self): + r = _valid() + r.rejection_gate = "manual_review" + errors = [e for e in r.validate() if "rejection_gate" in e] + assert errors == [] + + def test_empty_evidence_summary_fails(self): + r = _valid() + r.evidence_summary = "" + assert any("evidence_summary" in e for e in r.validate()) + + +# --------------------------------------------------------------------------- +# Group 5: ProofLadderAndDryLabRules (10 tests) +# --------------------------------------------------------------------------- + +class TestProofLadderAndDryLabRules: + def test_invalid_ladder_level_fails(self): + r = _valid() + r.proof_ladder_level_at_rejection = "not_a_level" + assert any("proof_ladder_level" in e for e in r.validate()) + + def test_all_dry_lab_safe_levels_pass(self): + safe_levels = VALID_PROOF_LADDER_LEVELS[:5] # up to multi_signal + for level in safe_levels: + r = _valid() + r.proof_ladder_level_at_rejection = level + errors = [e for e in r.validate() if "proof_ladder_level" in e] + assert errors == [], f"Level {level!r} should be valid for dry_lab_only" + + def test_dry_lab_only_false_fails(self): + r = _valid() + r.dry_lab_only = False + assert any("dry_lab_only" in e for e in r.validate()) + + def test_dry_lab_cap_enforced(self): + r = _valid() + r.proof_ladder_level_at_rejection = "expert_reviewed_assay_proposal" # level 5 + assert any("cap" in e or "dry_lab" in e for e in r.validate()) + + def test_dry_lab_cap_at_multi_signal(self): + r = _valid() + r.proof_ladder_level_at_rejection = "multi_signal_candidate_evidence" + errors = [e for e in r.validate() if "proof_ladder_level" in e] + assert errors == [] + + def test_valid_input_level_passes(self): + r = _valid() + r.proof_ladder_level_at_rejection = "valid_input" + errors = [e for e in r.validate() if "proof_ladder_level" in e] + assert errors == [] + + def test_leakage_aware_benchmark_passes(self): + r = _valid() + r.proof_ladder_level_at_rejection = "leakage_aware_benchmark" + errors = [e for e in r.validate() if "proof_ladder_level" in e] + assert errors == [] + + def test_independent_replication_blocked(self): + r = _valid() + r.proof_ladder_level_at_rejection = "independent_replication" + assert any("dry_lab" in e or "cap" in e for e in r.validate()) + + def test_reusable_discovery_loop_blocked(self): + r = _valid() + r.proof_ladder_level_at_rejection = "reusable_discovery_loop" + assert any("dry_lab" in e or "cap" in e for e in r.validate()) + + def test_scores_not_dict_fails(self): + r = _valid() + r.scores = "not_a_dict" # type: ignore + assert any("scores" in e for e in r.validate()) + + +# --------------------------------------------------------------------------- +# Group 6: NotesAndWarnings (8 tests) +# --------------------------------------------------------------------------- + +class TestNotesAndWarnings: + def test_notes_exceeds_400_fails(self): + r = _valid() + r.notes = "x" * 401 + assert any("notes" in e for e in r.validate()) + + def test_notes_exactly_400_passes(self): + r = _valid() + r.notes = "x" * 400 + errors = [e for e in r.validate() if "notes" in e and "WARNING" not in e] + assert errors == [] + + def test_empty_notes_produces_warning(self): + r = _valid() + r.notes = "" + result = r.validate() + assert any("WARNING" in msg and "notes" in msg for msg in result) + + def test_short_evidence_summary_produces_warning(self): + r = _valid() + r.evidence_summary = "Too short." + result = r.validate() + assert any("WARNING" in msg and "evidence_summary" in msg for msg in result) + + def test_warnings_not_errors(self): + r = _valid() + r.notes = "" + result = r.validate() + assert all("WARNING" in msg for msg in result) + + def test_valid_with_notes_no_warning(self): + r = _valid() + r.notes = "Reviewed by domain expert." + result = r.validate() + assert not any("notes" in msg for msg in result) + + def test_empty_scores_dict_passes(self): + r = _valid() + r.scores = {} + assert r.validate() == [] + + def test_multiple_scores_pass(self): + r = _valid() + r.scores = {"activity": 0.31, "safety": 0.85, "novelty": 0.55, "ensemble": 0.40} + assert r.validate() == [] + + +# --------------------------------------------------------------------------- +# Group 7: ValidateDictEdgeCases (9 tests) +# --------------------------------------------------------------------------- + +class TestValidateDictEdgeCases: + def test_empty_dict_fails(self): + errors = validate_dict({}) + assert len(errors) > 0 + + def test_missing_crc_id_fails(self): + d = _valid_dict() + del d["crc_id"] + errors = validate_dict(d) + assert any("crc_id" in e for e in errors) + + def test_missing_dry_lab_only_defaults_false(self): + d = _valid_dict() + del d["dry_lab_only"] + errors = validate_dict(d) + assert any("dry_lab_only" in e for e in errors) + + def test_missing_scores_defaults_empty_dict(self): + d = _valid_dict() + del d["scores"] + # scores defaults to {} in validate_dict + errors = [e for e in validate_dict(d) if "scores" in e and "WARNING" not in e] + assert errors == [] + + def test_missing_notes_defaults_empty_string(self): + d = _valid_dict() + del d["notes"] + result = validate_dict(d) + # empty notes → warning only + assert all("WARNING" in msg for msg in result) or result == [] + + def test_construction_error_returns_error(self): + result = validate_dict({"crc_id": None, "pipeline_version": None}) + assert len(result) > 0 + + def test_validate_dict_valid_returns_empty(self): + assert validate_dict(_valid_dict()) == [] + + def test_validate_dict_bad_gate(self): + d = _valid_dict() + d["rejection_gate"] = "made_up_gate" + errors = validate_dict(d) + assert any("rejection_gate" in e for e in errors) + + def test_validate_dict_bad_reason(self): + d = _valid_dict() + d["rejection_reason"] = "made_up_reason" + errors = validate_dict(d) + assert any("rejection_reason" in e for e in errors) diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index b7cd00ff..530ea30c 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 6520 +BASELINE = 6583 TOLERANCE = 0.05 # allow 5% fluctuation