diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index a3a120f1..4e34a1cb 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -9,6 +9,14 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak ## Changelog +### v0.10.29 — Phase B B5: Certificate quality-tier validator +- Added `src/openamp_foundry/evidence/certificate_quality.py` — assess_certificate_quality() function +- Three tiers: draft (candidate_id+sequence+scores) → internal_review (+selection_reason+known_failure_modes+proof_ladder_level+baseline_caveat+pipeline_version, no forbidden claims) → external_review_ready (+recommended_next_steps+references_checked+config_hash, no warnings) +- Returns: quality_tier, missing_fields, claim_violations, warnings, is_external_review_ready +- 63 tests in `tests/test_certificate_quality_validator.py` +- BASELINE 6394→6457 +- Closes Phase B B5 — external-review readiness is now machine-verifiable, not reviewer-dependent + ### v0.10.28 — Phase B B3: baseline_caveat field in certificates - Added `baseline_caveat` field to `build_certificate()` in `src/openamp_foundry/evidence/certificate.py` - Auto-computes three cheap-baseline flags: charge≥4, length 10-40aa, hydrophobic_fraction≥0.30 diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 344cee6e..756a5181 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -45,7 +45,7 @@ Make candidate evidence packages impossible to confuse with biological proof. | B2 | Add `unsupported_claims` field to certificates (complete). | CCB- schema: 10 fields, 13 validation rules, 8 claim class vocabulary, ≥3 classes required, dry_lab_only+all_listed_classes_unsupported enforced, no-duplicate check; negative complement of PLC- closes score-to-proof drift gap. | B | | B3 | Add `baseline_caveat` field (complete). | Added to build_certificate(): auto-computes charge/length/hydrophobicity flags and warns when all three cheap baselines pass; forces cheap-explanation visibility at certificate issue time. 63 tests in tests/test_certificate_baseline_caveat.py. | B | | B4 | Add `release_status` field. | Supports staged release. | C/D | -| B5 | Add certificate quality-tier validator. | External-review readiness becomes measurable. | B | +| B5 | Add certificate quality-tier validator (complete). | assess_certificate_quality() computes draft/internal_review/external_review_ready tier; missing-field detection, forbidden-claim violation check, external-review gate; 63 tests in tests/test_certificate_quality_validator.py. | B | | B6 | Add human-readable certificate report. | Domain experts can inspect faster. | B | | B7 | Add candidate rejection certificate support. | Failures become artifacts. | B | | B8 | Link certificates to run manifest hashes. | Stronger reproducibility. | B | diff --git a/src/openamp_foundry/evidence/certificate_quality.py b/src/openamp_foundry/evidence/certificate_quality.py new file mode 100644 index 00000000..4973c6f6 --- /dev/null +++ b/src/openamp_foundry/evidence/certificate_quality.py @@ -0,0 +1,149 @@ +"""Certificate quality-tier validator — Phase B B5. + +Computes a quality tier for a certificate dict produced by build_certificate(). +Makes external-review readiness a computed, machine-verifiable property rather +than a reviewer judgment call. + +Three tiers (ordered ascending): + draft — candidate_id, sequence, scores present + internal_review — draft + selection_reason, known_failure_modes, + proof_ladder_level, baseline_caveat, pipeline_version; + zero forbidden-claim violations + external_review_ready — internal_review + recommended_next_steps, + references_checked, config_hash; zero warnings +""" + +from __future__ import annotations + +import re +from typing import Any + +# Fields required for each tier +_DRAFT_REQUIRED = frozenset({"candidate_id", "sequence", "scores"}) + +_INTERNAL_REQUIRED = frozenset({ + "candidate_id", "sequence", "scores", + "selection_reason", "known_failure_modes", + "proof_ladder_level", "baseline_caveat", "pipeline_version", +}) + +_EXTERNAL_REQUIRED = frozenset({ + "candidate_id", "sequence", "scores", + "selection_reason", "known_failure_modes", + "proof_ladder_level", "baseline_caveat", "pipeline_version", + "recommended_next_steps", "references_checked", "config_hash", +}) + +# Ordered tiers (ascending quality) +QUALITY_TIERS = ("draft", "internal_review", "external_review_ready") + +# Simple forbidden-claim patterns (subset safe to import here without circular dep) +_FORBIDDEN_PHRASES = ( + re.compile(r"\bproven\b", re.IGNORECASE), + re.compile(r"safe in humans", re.IGNORECASE), + re.compile(r"clinical(ly)?\b", re.IGNORECASE), + re.compile(r"\bdrug candidate\b", re.IGNORECASE), + re.compile(r"\bcure\b", re.IGNORECASE), + re.compile(r"antibiotic for treatment", re.IGNORECASE), + re.compile(r"wet.lab evidence confirms", re.IGNORECASE), +) + + +def _extract_text_fields(cert: dict[str, Any]) -> list[str]: + texts: list[str] = [] + for val in cert.values(): + if isinstance(val, str): + texts.append(val) + elif isinstance(val, list): + for item in val: + if isinstance(item, str): + texts.append(item) + return texts + + +def _find_claim_violations(cert: dict[str, Any]) -> list[str]: + violations: list[str] = [] + texts = _extract_text_fields(cert) + for pat in _FORBIDDEN_PHRASES: + for text in texts: + if pat.search(text): + violations.append( + f"ForbiddenPattern {pat.pattern!r} found in: {text[:60]!r}" + ) + return violations + + +def _missing_fields(cert: dict[str, Any], required: frozenset[str]) -> list[str]: + missing = [] + for field in sorted(required): + val = cert.get(field) + if val is None: + missing.append(field) + elif isinstance(val, str) and not val.strip(): + missing.append(field) + elif isinstance(val, list) and len(val) == 0: + missing.append(field) + return missing + + +def assess_certificate_quality(cert: dict[str, Any]) -> dict[str, Any]: + """Assess the quality tier of a certificate dict. + + Returns a quality report dict with: + quality_tier: str — one of QUALITY_TIERS + missing_fields: list[str] — fields absent or empty at the assessed tier + claim_violations: list[str] — forbidden-claim pattern matches + warnings: list[str] — non-blocking issues (e.g. empty optional fields) + is_external_review_ready: bool + + The tier is the highest tier all requirements are met for. + If draft requirements are not met, quality_tier is "below_draft". + """ + claim_violations = _find_claim_violations(cert) + warnings: list[str] = [] + + # Check draft + draft_missing = _missing_fields(cert, _DRAFT_REQUIRED) + if draft_missing: + return { + "quality_tier": "below_draft", + "missing_fields": draft_missing, + "claim_violations": claim_violations, + "warnings": [f"Missing draft-required field: {f}" for f in draft_missing], + "is_external_review_ready": False, + } + + # Check internal_review + internal_missing = _missing_fields(cert, _INTERNAL_REQUIRED) + if internal_missing or claim_violations: + if claim_violations: + warnings.append(f"{len(claim_violations)} forbidden-claim violation(s) block internal_review tier") + return { + "quality_tier": "draft", + "missing_fields": internal_missing, + "claim_violations": claim_violations, + "warnings": warnings + [f"Missing internal-required field: {f}" for f in internal_missing], + "is_external_review_ready": False, + } + + # Check external_review_ready + external_missing = _missing_fields(cert, _EXTERNAL_REQUIRED) + if external_missing: + for f in external_missing: + warnings.append(f"Missing external-required field: {f}") + return { + "quality_tier": "internal_review", + "missing_fields": external_missing, + "claim_violations": [], + "warnings": warnings, + "is_external_review_ready": False, + } + + # All tiers met + return { + "quality_tier": "external_review_ready", + "missing_fields": [], + "claim_violations": [], + "warnings": [], + "is_external_review_ready": True, + } diff --git a/tests/test_certificate_quality_validator.py b/tests/test_certificate_quality_validator.py new file mode 100644 index 00000000..463d39f8 --- /dev/null +++ b/tests/test_certificate_quality_validator.py @@ -0,0 +1,394 @@ +"""Tests for certificate quality-tier validator — Phase B B5. + +63 tests across 7 groups verifying that assess_certificate_quality() correctly +computes draft / internal_review / external_review_ready tiers and that +missing fields and forbidden claims are correctly detected. +""" + +from __future__ import annotations + +import pytest + +from openamp_foundry.evidence.certificate import build_certificate +from openamp_foundry.evidence.certificate_quality import ( + QUALITY_TIERS, + _DRAFT_REQUIRED, + _EXTERNAL_REQUIRED, + _INTERNAL_REQUIRED, + assess_certificate_quality, +) +from openamp_foundry.features.physchem import compute_features +from openamp_foundry.types import PeptideCandidate, ScoredCandidate + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_SEQ = "KWKLFKKIGAVLKVL" + + +def _make_scored(sequence: str = _SEQ) -> ScoredCandidate: + return ScoredCandidate( + candidate=PeptideCandidate("AMPF-001", sequence, "test_source"), + features=compute_features(sequence), + scores={"activity": 0.80, "safety": 0.90, "novelty": 0.50, "ensemble": 0.82}, + nearest_reference=None, + selection_reason=["high ensemble score"], + known_failure_modes=["No wet-lab assay has been run."], + ) + + +def _full_cert() -> dict: + return build_certificate(_make_scored(), {"threshold": 0.7}, ["APD-001"]) + + +def _minimal_cert() -> dict: + return { + "candidate_id": "AMPF-001", + "sequence": _SEQ, + "scores": {"activity": 0.80}, + } + + +# --------------------------------------------------------------------------- +# Group 1: TierConstants (6 tests) +# --------------------------------------------------------------------------- + +class TestTierConstants: + """QUALITY_TIERS is ordered and complete.""" + + def test_quality_tiers_is_tuple(self): + assert isinstance(QUALITY_TIERS, tuple) + + def test_quality_tiers_has_three_values(self): + assert len(QUALITY_TIERS) == 3 + + def test_quality_tiers_draft_first(self): + assert QUALITY_TIERS[0] == "draft" + + def test_quality_tiers_internal_review_second(self): + assert QUALITY_TIERS[1] == "internal_review" + + def test_quality_tiers_external_review_ready_third(self): + assert QUALITY_TIERS[2] == "external_review_ready" + + def test_draft_required_fields_subset_of_internal(self): + assert _DRAFT_REQUIRED.issubset(_INTERNAL_REQUIRED) + + +# --------------------------------------------------------------------------- +# Group 2: FullCertificateIsExternalReady (10 tests) +# --------------------------------------------------------------------------- + +class TestFullCertificateIsExternalReady: + """build_certificate() output reaches external_review_ready tier.""" + + def test_full_cert_tier_is_external_review_ready(self): + report = assess_certificate_quality(_full_cert()) + assert report["quality_tier"] == "external_review_ready" + + def test_full_cert_is_external_review_ready_true(self): + report = assess_certificate_quality(_full_cert()) + assert report["is_external_review_ready"] is True + + def test_full_cert_no_missing_fields(self): + report = assess_certificate_quality(_full_cert()) + assert report["missing_fields"] == [] + + def test_full_cert_no_claim_violations(self): + report = assess_certificate_quality(_full_cert()) + assert report["claim_violations"] == [] + + def test_full_cert_no_warnings(self): + report = assess_certificate_quality(_full_cert()) + assert report["warnings"] == [] + + def test_report_has_quality_tier_key(self): + report = assess_certificate_quality(_full_cert()) + assert "quality_tier" in report + + def test_report_has_missing_fields_key(self): + report = assess_certificate_quality(_full_cert()) + assert "missing_fields" in report + + def test_report_has_claim_violations_key(self): + report = assess_certificate_quality(_full_cert()) + assert "claim_violations" in report + + def test_report_has_warnings_key(self): + report = assess_certificate_quality(_full_cert()) + assert "warnings" in report + + def test_report_has_is_external_review_ready_key(self): + report = assess_certificate_quality(_full_cert()) + assert "is_external_review_ready" in report + + +# --------------------------------------------------------------------------- +# Group 3: MinimalCertIsDraft (10 tests) +# --------------------------------------------------------------------------- + +class TestMinimalCertIsDraft: + """Minimal cert with only draft fields reaches draft tier.""" + + def test_minimal_cert_tier_is_draft(self): + report = assess_certificate_quality(_minimal_cert()) + assert report["quality_tier"] == "draft" + + def test_minimal_cert_not_external_review_ready(self): + report = assess_certificate_quality(_minimal_cert()) + assert report["is_external_review_ready"] is False + + def test_minimal_cert_missing_internal_fields(self): + report = assess_certificate_quality(_minimal_cert()) + assert len(report["missing_fields"]) > 0 + + def test_minimal_cert_missing_selection_reason(self): + report = assess_certificate_quality(_minimal_cert()) + assert "selection_reason" in report["missing_fields"] + + def test_minimal_cert_missing_proof_ladder_level(self): + report = assess_certificate_quality(_minimal_cert()) + assert "proof_ladder_level" in report["missing_fields"] + + def test_minimal_cert_missing_baseline_caveat(self): + report = assess_certificate_quality(_minimal_cert()) + assert "baseline_caveat" in report["missing_fields"] + + def test_minimal_cert_missing_pipeline_version(self): + report = assess_certificate_quality(_minimal_cert()) + assert "pipeline_version" in report["missing_fields"] + + def test_minimal_cert_no_claim_violations(self): + report = assess_certificate_quality(_minimal_cert()) + assert report["claim_violations"] == [] + + def test_minimal_cert_has_warnings(self): + report = assess_certificate_quality(_minimal_cert()) + assert len(report["warnings"]) > 0 + + def test_minimal_cert_not_in_quality_tiers_tuple(self): + report = assess_certificate_quality(_minimal_cert()) + # draft is in QUALITY_TIERS + assert report["quality_tier"] in ("draft", "below_draft") or report["quality_tier"] in QUALITY_TIERS + + +# --------------------------------------------------------------------------- +# Group 4: BelowDraft (8 tests) +# --------------------------------------------------------------------------- + +class TestBelowDraft: + """Certs missing draft-required fields are below_draft.""" + + def test_empty_cert_is_below_draft(self): + report = assess_certificate_quality({}) + assert report["quality_tier"] == "below_draft" + + def test_missing_candidate_id_is_below_draft(self): + cert = {"sequence": _SEQ, "scores": {}} + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "below_draft" + + def test_missing_sequence_is_below_draft(self): + cert = {"candidate_id": "AMPF-001", "scores": {}} + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "below_draft" + + def test_missing_scores_is_below_draft(self): + cert = {"candidate_id": "AMPF-001", "sequence": _SEQ} + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "below_draft" + + def test_below_draft_not_external_review_ready(self): + report = assess_certificate_quality({}) + assert report["is_external_review_ready"] is False + + def test_below_draft_has_missing_fields(self): + report = assess_certificate_quality({}) + assert len(report["missing_fields"]) > 0 + + def test_empty_candidate_id_is_below_draft(self): + cert = {"candidate_id": "", "sequence": _SEQ, "scores": {}} + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "below_draft" + + def test_empty_sequence_is_below_draft(self): + cert = {"candidate_id": "AMPF-001", "sequence": " ", "scores": {}} + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "below_draft" + + +# --------------------------------------------------------------------------- +# Group 5: ClaimViolationsBlockTier (10 tests) +# --------------------------------------------------------------------------- + +class TestClaimViolationsBlockTier: + """Forbidden-claim violations block tier advancement past draft.""" + + def _cert_with_claim(self, claim: str) -> dict: + cert = _full_cert() + cert["notes"] = claim + return cert + + def test_proven_claim_blocks_internal_review(self): + cert = self._cert_with_claim("This candidate is proven to be active.") + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "draft" + + def test_clinical_claim_blocks_internal_review(self): + cert = self._cert_with_claim("Clinically useful compound.") + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "draft" + + def test_drug_candidate_claim_blocks(self): + cert = self._cert_with_claim("This is a drug candidate.") + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "draft" + + def test_cure_claim_blocks(self): + cert = self._cert_with_claim("A cure for MRSA infections.") + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "draft" + + def test_safe_in_humans_claim_blocks(self): + cert = self._cert_with_claim("Safe in humans.") + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "draft" + + def test_antibiotic_for_treatment_blocks(self): + cert = self._cert_with_claim("Antibiotic for treatment of MRSA.") + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "draft" + + def test_claim_violation_is_in_report(self): + cert = self._cert_with_claim("proven active compound") + report = assess_certificate_quality(cert) + assert len(report["claim_violations"]) > 0 + + def test_claim_violation_has_pattern_info(self): + cert = self._cert_with_claim("This is proven.") + report = assess_certificate_quality(cert) + assert any("proven" in v.lower() for v in report["claim_violations"]) + + def test_clean_cert_has_no_claim_violations(self): + report = assess_certificate_quality(_full_cert()) + assert report["claim_violations"] == [] + + def test_dry_lab_language_not_a_violation(self): + cert = _full_cert() + cert["notes"] = "Computationally nominated dry-lab candidate only." + report = assess_certificate_quality(cert) + assert report["claim_violations"] == [] + + +# --------------------------------------------------------------------------- +# Group 6: InternalReviewTier (10 tests) +# --------------------------------------------------------------------------- + +class TestInternalReviewTier: + """Cert with all internal fields but missing external fields reaches internal_review.""" + + def _internal_cert(self) -> dict: + """Full cert minus the external-only fields.""" + cert = _full_cert() + # Remove external-only fields (those in _EXTERNAL_REQUIRED but not _INTERNAL_REQUIRED) + for field in list(_EXTERNAL_REQUIRED - _INTERNAL_REQUIRED): + cert.pop(field, None) + return cert + + def test_internal_cert_tier_is_internal_review(self): + report = assess_certificate_quality(self._internal_cert()) + assert report["quality_tier"] == "internal_review" + + def test_internal_cert_not_external_review_ready(self): + report = assess_certificate_quality(self._internal_cert()) + assert report["is_external_review_ready"] is False + + def test_internal_cert_has_missing_external_fields(self): + report = assess_certificate_quality(self._internal_cert()) + assert len(report["missing_fields"]) > 0 + + def test_internal_cert_no_claim_violations(self): + report = assess_certificate_quality(self._internal_cert()) + assert report["claim_violations"] == [] + + def test_internal_cert_warnings_mention_missing_external(self): + report = assess_certificate_quality(self._internal_cert()) + assert any("external" in w.lower() for w in report["warnings"]) + + def test_internal_required_is_subset_of_external_required(self): + assert _INTERNAL_REQUIRED.issubset(_EXTERNAL_REQUIRED) + + def test_external_has_more_fields_than_internal(self): + assert len(_EXTERNAL_REQUIRED) > len(_INTERNAL_REQUIRED) + + def test_missing_recommended_next_steps_gives_internal_tier(self): + cert = _full_cert() + cert.pop("recommended_next_steps", None) + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "internal_review" + + def test_missing_config_hash_gives_internal_tier(self): + cert = _full_cert() + cert.pop("config_hash", None) + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "internal_review" + + def test_empty_references_checked_gives_internal_tier(self): + cert = _full_cert() + cert["references_checked"] = [] + report = assess_certificate_quality(cert) + # Empty list counts as missing + assert report["quality_tier"] == "internal_review" + + +# --------------------------------------------------------------------------- +# Group 7: EdgeCases (9 tests) +# --------------------------------------------------------------------------- + +class TestEdgeCases: + """Edge cases and boundary conditions.""" + + def test_assess_returns_dict(self): + report = assess_certificate_quality(_full_cert()) + assert isinstance(report, dict) + + def test_missing_fields_is_list(self): + report = assess_certificate_quality(_full_cert()) + assert isinstance(report["missing_fields"], list) + + def test_claim_violations_is_list(self): + report = assess_certificate_quality(_full_cert()) + assert isinstance(report["claim_violations"], list) + + def test_warnings_is_list(self): + report = assess_certificate_quality(_full_cert()) + assert isinstance(report["warnings"], list) + + def test_is_external_review_ready_is_bool(self): + report = assess_certificate_quality(_full_cert()) + assert isinstance(report["is_external_review_ready"], bool) + + def test_cert_with_none_field_treated_as_missing(self): + cert = _minimal_cert() + cert["selection_reason"] = None + # None counts as missing, so still draft at best + report = assess_certificate_quality(cert) + assert "selection_reason" in report.get("missing_fields", []) or report["quality_tier"] in ("draft", "below_draft") + + def test_cert_with_empty_string_field_treated_as_missing(self): + cert = dict(_full_cert()) + cert["pipeline_version"] = "" + report = assess_certificate_quality(cert) + assert report["quality_tier"] != "external_review_ready" + + def test_below_draft_quality_tier_string(self): + report = assess_certificate_quality({}) + assert isinstance(report["quality_tier"], str) + + def test_claim_in_list_field_detected(self): + cert = _full_cert() + cert["selection_reason"] = ["This is a proven cure for infections."] + report = assess_certificate_quality(cert) + assert report["quality_tier"] == "draft" diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 046deb9e..26e73559 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 6394 +BASELINE = 6457 TOLERANCE = 0.05 # allow 5% fluctuation