diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 4e34a1cb..3745006a 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -9,6 +9,15 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak ## Changelog +### v0.10.30 — Phase B B6: Human-readable certificate report +- Added `src/openamp_foundry/evidence/certificate_report.py` — build_certificate_report() function +- Converts cert dict to formatted text with labelled sections: CANDIDATE, PROOF LADDER, SCORES (with not-biological-proof notice), CHEAP-EXPLANATION CHECK, SELECTION REASON, KNOWN FAILURE MODES, RECOMMENDED NEXT STEPS, REFERENCES CHECKED, optional QUALITY TIER +- Footer: DRY-LAB COMPUTATIONAL OUTPUTS ONLY notice on every report +- Optional quality_report parameter integrates with assess_certificate_quality() +- 63 tests in `tests/test_certificate_report.py` +- BASELINE 6457→6520 +- Closes Phase B B6 — domain experts can now inspect candidate certificates as readable text + ### v0.10.29 — Phase B B5: Certificate quality-tier validator - Added `src/openamp_foundry/evidence/certificate_quality.py` — assess_certificate_quality() function - Three tiers: draft (candidate_id+sequence+scores) → internal_review (+selection_reason+known_failure_modes+proof_ladder_level+baseline_caveat+pipeline_version, no forbidden claims) → external_review_ready (+recommended_next_steps+references_checked+config_hash, no warnings) diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 756a5181..d6b4d849 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -46,7 +46,7 @@ Make candidate evidence packages impossible to confuse with biological proof. | B3 | Add `baseline_caveat` field (complete). | Added to build_certificate(): auto-computes charge/length/hydrophobicity flags and warns when all three cheap baselines pass; forces cheap-explanation visibility at certificate issue time. 63 tests in tests/test_certificate_baseline_caveat.py. | B | | B4 | Add `release_status` field. | Supports staged release. | C/D | | B5 | Add certificate quality-tier validator (complete). | assess_certificate_quality() computes draft/internal_review/external_review_ready tier; missing-field detection, forbidden-claim violation check, external-review gate; 63 tests in tests/test_certificate_quality_validator.py. | B | -| B6 | Add human-readable certificate report. | Domain experts can inspect faster. | B | +| B6 | Add human-readable certificate report (complete). | build_certificate_report() in certificate_report.py converts cert dict to formatted text with sections: identity, proof ladder, scores, cheap-explanation check, selection reason, failure modes, next steps, references, optional quality tier; 63 tests in tests/test_certificate_report.py. | B | | B7 | Add candidate rejection certificate support. | Failures become artifacts. | B | | B8 | Link certificates to run manifest hashes. | Stronger reproducibility. | B | | B9 | Add test that dry-lab certificates cannot include forbidden claims (complete). | CI gate: tests/test_certificate_claim_discipline.py scans all cert text fields against RISKY_PATTERNS and FORBIDDEN_CLAIM_PATTERNS; default cert proven clean; forbidden phrases confirmed detectable. 35 tests total. | B/C | diff --git a/src/openamp_foundry/evidence/certificate_report.py b/src/openamp_foundry/evidence/certificate_report.py new file mode 100644 index 00000000..9f9e116a --- /dev/null +++ b/src/openamp_foundry/evidence/certificate_report.py @@ -0,0 +1,170 @@ +"""Human-readable certificate report generator — Phase B B6. + +Converts a certificate dict (from build_certificate()) into a formatted text +report that domain experts can read quickly without parsing JSON. + +Usage: + from openamp_foundry.evidence.certificate import build_certificate + from openamp_foundry.evidence.certificate_report import build_certificate_report + + cert = build_certificate(scored, config, references) + report = build_certificate_report(cert) + print(report) +""" + +from __future__ import annotations + +from typing import Any + +_SEPARATOR = "=" * 72 +_SECTION_SEP = "-" * 72 + + +def _fmt_score(val: Any) -> str: + if isinstance(val, float): + return f"{val:.3f}" + return str(val) + + +def _fmt_list(items: Any, indent: int = 2) -> str: + prefix = " " * indent + if not items: + return f"{prefix}(none)" + if isinstance(items, list): + return "\n".join(f"{prefix}• {item}" for item in items) + return f"{prefix}{items}" + + +def build_certificate_report( + cert: dict[str, Any], + quality_report: dict[str, Any] | None = None, +) -> str: + """Build a human-readable text report from a certificate dict. + + Args: + cert: Certificate dict from build_certificate(). + quality_report: Optional output of assess_certificate_quality(cert). + If provided, quality tier and readiness are included. + + Returns: + A multi-line string formatted for domain expert review. + """ + lines: list[str] = [] + + # Header + lines.append(_SEPARATOR) + lines.append("OPENAMP FOUNDRY — CANDIDATE EVIDENCE CERTIFICATE") + lines.append(_SEPARATOR) + lines.append("") + + # Identity + lines.append("CANDIDATE") + lines.append(_SECTION_SEP) + lines.append(f" ID: {cert.get('candidate_id', '(unknown)')}") + lines.append(f" Sequence: {cert.get('sequence', '(none)')}") + lines.append(f" Source: {cert.get('source', '(unknown)')}") + lines.append(f" Version: {cert.get('pipeline_version', '(unknown)')}") + lines.append(f" Generated:{cert.get('generated_at', '(unknown)')}") + lines.append("") + + # Proof ladder + ladder_level = cert.get("proof_ladder_level", "(not set)") + lines.append("PROOF LADDER") + lines.append(_SECTION_SEP) + lines.append(f" Level: {ladder_level}") + lines.append("") + + # Scores + lines.append("SCORES [dry-lab computational — not biological proof]") + lines.append(_SECTION_SEP) + scores = cert.get("scores", {}) + if isinstance(scores, dict): + for key, val in sorted(scores.items()): + lines.append(f" {key:<20} {_fmt_score(val)}") + else: + lines.append(" (none)") + lines.append("") + + # Baseline caveat + lines.append("CHEAP-EXPLANATION CHECK") + lines.append(_SECTION_SEP) + caveat = cert.get("baseline_caveat", "(not computed)") + for chunk in _wrap(caveat, width=68, indent=2): + lines.append(chunk) + lines.append("") + + # Selection reason + lines.append("SELECTION REASON") + lines.append(_SECTION_SEP) + lines.append(_fmt_list(cert.get("selection_reason", []))) + lines.append("") + + # Known failure modes + lines.append("KNOWN FAILURE MODES") + lines.append(_SECTION_SEP) + lines.append(_fmt_list(cert.get("known_failure_modes", []))) + lines.append("") + + # Recommended next steps + lines.append("RECOMMENDED NEXT STEPS") + lines.append(_SECTION_SEP) + lines.append(_fmt_list(cert.get("recommended_next_steps", []))) + lines.append("") + + # References checked + refs = cert.get("references_checked", []) + lines.append("REFERENCES CHECKED") + lines.append(_SECTION_SEP) + lines.append(_fmt_list(refs) if refs else " (none)") + lines.append("") + + # Quality tier (optional) + if quality_report is not None: + tier = quality_report.get("quality_tier", "(unknown)") + is_ready = quality_report.get("is_external_review_ready", False) + missing = quality_report.get("missing_fields", []) + violations = quality_report.get("claim_violations", []) + warnings = quality_report.get("warnings", []) + + lines.append("QUALITY TIER") + lines.append(_SECTION_SEP) + lines.append(f" Tier: {tier}") + lines.append(f" External-review ready: {'YES' if is_ready else 'NO'}") + if missing: + lines.append(f" Missing fields: {', '.join(missing)}") + if violations: + lines.append(f" Claim violations: {len(violations)}") + if warnings: + for w in warnings: + lines.append(f" Warning: {w}") + lines.append("") + + # Footer + lines.append(_SEPARATOR) + lines.append("NOTICE: This certificate reflects DRY-LAB COMPUTATIONAL OUTPUTS ONLY.") + lines.append("No biological activity has been confirmed. All scores are predictions.") + lines.append("Independent domain expert review is required before any assay decision.") + lines.append(_SEPARATOR) + + return "\n".join(lines) + + +def _wrap(text: str, width: int = 68, indent: int = 2) -> list[str]: + """Simple word-wrap that preserves leading indent.""" + prefix = " " * indent + words = text.split() + if not words: + return [prefix] + lines: list[str] = [] + current = prefix + for word in words: + candidate = current + (" " if current.strip() else "") + word + if len(candidate) <= width: + current = candidate + else: + if current.strip(): + lines.append(current) + current = prefix + word + if current.strip(): + lines.append(current) + return lines if lines else [prefix] diff --git a/tests/test_certificate_report.py b/tests/test_certificate_report.py new file mode 100644 index 00000000..f79a3062 --- /dev/null +++ b/tests/test_certificate_report.py @@ -0,0 +1,405 @@ +"""Tests for human-readable certificate report — Phase B B6. + +63 tests across 7 groups verifying that build_certificate_report() produces +correct, claim-clean, readable text output for domain experts. +""" + +from __future__ import annotations + +import pytest + +from openamp_foundry.evidence.certificate import build_certificate +from openamp_foundry.evidence.certificate_quality import assess_certificate_quality +from openamp_foundry.evidence.certificate_report import ( + _wrap, + build_certificate_report, +) +from openamp_foundry.features.physchem import compute_features +from openamp_foundry.types import PeptideCandidate, ScoredCandidate + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_SEQ = "KWKLFKKIGAVLKVL" + + +def _make_scored(sequence: str = _SEQ) -> ScoredCandidate: + return ScoredCandidate( + candidate=PeptideCandidate("AMPF-001", sequence, "test_source"), + features=compute_features(sequence), + scores={"activity": 0.80, "safety": 0.90, "novelty": 0.50, "ensemble": 0.82}, + nearest_reference=None, + selection_reason=["high ensemble score"], + known_failure_modes=["No wet-lab assay has been run."], + ) + + +def _cert() -> dict: + return build_certificate(_make_scored(), {"threshold": 0.7}, ["APD-001"]) + + +# --------------------------------------------------------------------------- +# Group 1: ReturnType (6 tests) +# --------------------------------------------------------------------------- + +class TestReturnType: + """build_certificate_report() returns a non-empty string.""" + + def test_returns_string(self): + assert isinstance(build_certificate_report(_cert()), str) + + def test_returns_non_empty(self): + assert len(build_certificate_report(_cert())) > 0 + + def test_returns_multiline(self): + assert "\n" in build_certificate_report(_cert()) + + def test_minimal_cert_returns_string(self): + cert = {"candidate_id": "X", "sequence": "AAA", "scores": {}} + assert isinstance(build_certificate_report(cert), str) + + def test_empty_cert_returns_string(self): + assert isinstance(build_certificate_report({}), str) + + def test_with_quality_report_returns_string(self): + cert = _cert() + qr = assess_certificate_quality(cert) + assert isinstance(build_certificate_report(cert, quality_report=qr), str) + + +# --------------------------------------------------------------------------- +# Group 2: HeaderAndFooter (8 tests) +# --------------------------------------------------------------------------- + +class TestHeaderAndFooter: + """Report has correct header and footer.""" + + def test_header_present(self): + report = build_certificate_report(_cert()) + assert "OPENAMP FOUNDRY" in report + + def test_candidate_evidence_certificate_in_header(self): + report = build_certificate_report(_cert()) + assert "CANDIDATE EVIDENCE CERTIFICATE" in report + + def test_separator_present(self): + report = build_certificate_report(_cert()) + assert "=" * 20 in report + + def test_notice_in_footer(self): + report = build_certificate_report(_cert()) + assert "NOTICE" in report + + def test_dry_lab_in_footer(self): + report = build_certificate_report(_cert()) + assert "DRY-LAB" in report + + def test_no_biological_activity_confirmed_in_footer(self): + report = build_certificate_report(_cert()) + assert "No biological activity has been confirmed" in report + + def test_independent_domain_expert_in_footer(self): + report = build_certificate_report(_cert()) + assert "Independent domain expert review" in report + + def test_scores_not_biological_proof_note(self): + report = build_certificate_report(_cert()) + assert "not biological proof" in report + + +# --------------------------------------------------------------------------- +# Group 3: IdentityFields (10 tests) +# --------------------------------------------------------------------------- + +class TestIdentityFields: + """Report includes candidate identity fields.""" + + def test_candidate_id_in_report(self): + report = build_certificate_report(_cert()) + assert "AMPF-001" in report + + def test_sequence_in_report(self): + report = build_certificate_report(_cert()) + assert _SEQ in report + + def test_source_in_report(self): + report = build_certificate_report(_cert()) + assert "test_source" in report + + def test_pipeline_version_in_report(self): + report = build_certificate_report(_cert()) + assert "Version" in report + + def test_generated_at_in_report(self): + report = build_certificate_report(_cert()) + assert "Generated" in report + + def test_candidate_section_label(self): + report = build_certificate_report(_cert()) + assert "CANDIDATE" in report + + def test_proof_ladder_section_label(self): + report = build_certificate_report(_cert()) + assert "PROOF LADDER" in report + + def test_proof_ladder_level_value_in_report(self): + cert = _cert() + level = cert.get("proof_ladder_level", "") + report = build_certificate_report(cert) + assert level in report + + def test_different_candidate_id_shows_correctly(self): + scored = ScoredCandidate( + candidate=PeptideCandidate("CUSTOM-999", _SEQ, "src"), + features=compute_features(_SEQ), + scores={"activity": 0.5}, + nearest_reference=None, + selection_reason=["test"], + known_failure_modes=["test"], + ) + cert = build_certificate(scored, {}, []) + report = build_certificate_report(cert) + assert "CUSTOM-999" in report + + def test_config_hash_not_required_in_report(self): + # Config hash is a field but needn't appear as a labeled section + cert = _cert() + report = build_certificate_report(cert) + # Report is still valid without explicit config hash display + assert isinstance(report, str) and len(report) > 0 + + +# --------------------------------------------------------------------------- +# Group 4: ScoresSection (8 tests) +# --------------------------------------------------------------------------- + +class TestScoresSection: + """Report includes scores section with correct values.""" + + def test_scores_section_label(self): + report = build_certificate_report(_cert()) + assert "SCORES" in report + + def test_activity_score_in_report(self): + report = build_certificate_report(_cert()) + assert "activity" in report + + def test_safety_score_in_report(self): + report = build_certificate_report(_cert()) + assert "safety" in report + + def test_score_value_formatted(self): + report = build_certificate_report(_cert()) + assert "0.800" in report or "0.80" in report + + def test_scores_section_not_biological_proof(self): + report = build_certificate_report(_cert()) + assert "not biological proof" in report.lower() + + def test_empty_scores_handled(self): + cert = {"candidate_id": "X", "sequence": "AAA", "scores": {}} + report = build_certificate_report(cert) + assert "SCORES" in report + + def test_scores_section_before_footer(self): + report = build_certificate_report(_cert()) + scores_pos = report.index("SCORES") + notice_pos = report.index("NOTICE") + assert scores_pos < notice_pos + + def test_novelty_score_in_report(self): + report = build_certificate_report(_cert()) + assert "novelty" in report + + +# --------------------------------------------------------------------------- +# Group 5: BaselineCaveatSection (8 tests) +# --------------------------------------------------------------------------- + +class TestBaselineCaveatSection: + """Report includes baseline caveat section.""" + + def test_cheap_explanation_check_in_report(self): + report = build_certificate_report(_cert()) + assert "CHEAP-EXPLANATION CHECK" in report + + def test_caveat_text_in_report(self): + cert = _cert() + caveat = cert.get("baseline_caveat", "") + report = build_certificate_report(cert) + # At least part of the caveat should appear + assert "Cheapest-explanation" in report or "charge=" in report + + def test_caveat_before_selection_reason(self): + report = build_certificate_report(_cert()) + caveat_pos = report.index("CHEAP-EXPLANATION") + sel_pos = report.index("SELECTION REASON") + assert caveat_pos < sel_pos + + def test_selection_reason_section_label(self): + report = build_certificate_report(_cert()) + assert "SELECTION REASON" in report + + def test_selection_reason_value_in_report(self): + report = build_certificate_report(_cert()) + assert "high ensemble score" in report + + def test_known_failure_modes_section_label(self): + report = build_certificate_report(_cert()) + assert "KNOWN FAILURE MODES" in report + + def test_known_failure_modes_value_in_report(self): + report = build_certificate_report(_cert()) + assert "No wet-lab assay has been run" in report + + def test_recommended_next_steps_section_label(self): + report = build_certificate_report(_cert()) + assert "RECOMMENDED NEXT STEPS" in report + + +# --------------------------------------------------------------------------- +# Group 6: QualityTierSection (10 tests) +# --------------------------------------------------------------------------- + +class TestQualityTierSection: + """Quality tier section appears correctly when quality_report is provided.""" + + def test_quality_tier_section_absent_without_report(self): + report = build_certificate_report(_cert()) + assert "QUALITY TIER" not in report + + def test_quality_tier_section_present_with_report(self): + cert = _cert() + qr = assess_certificate_quality(cert) + report = build_certificate_report(cert, quality_report=qr) + assert "QUALITY TIER" in report + + def test_tier_value_in_report(self): + cert = _cert() + qr = assess_certificate_quality(cert) + report = build_certificate_report(cert, quality_report=qr) + assert qr["quality_tier"] in report + + def test_external_review_ready_yes(self): + cert = _cert() + qr = assess_certificate_quality(cert) + report = build_certificate_report(cert, quality_report=qr) + if qr["is_external_review_ready"]: + assert "YES" in report + + def test_external_review_ready_no(self): + cert = {"candidate_id": "X", "sequence": "AAA", "scores": {}} + qr = assess_certificate_quality(cert) + report = build_certificate_report(cert, quality_report=qr) + assert "NO" in report + + def test_missing_fields_shown_in_report(self): + cert = {"candidate_id": "X", "sequence": "AAA", "scores": {}} + qr = assess_certificate_quality(cert) + report = build_certificate_report(cert, quality_report=qr) + assert len(qr["missing_fields"]) > 0 # sanity + for field in qr["missing_fields"]: + assert field in report + + def test_claim_violations_count_shown(self): + cert = _cert() + cert["notes"] = "This is proven." + qr = assess_certificate_quality(cert) + report = build_certificate_report(cert, quality_report=qr) + if qr["claim_violations"]: + assert "Claim violations" in report or "violations" in report.lower() + + def test_quality_section_before_footer(self): + cert = _cert() + qr = assess_certificate_quality(cert) + report = build_certificate_report(cert, quality_report=qr) + tier_pos = report.index("QUALITY TIER") + notice_pos = report.index("NOTICE") + assert tier_pos < notice_pos + + def test_no_quality_report_still_has_footer(self): + report = build_certificate_report(_cert()) + assert "NOTICE" in report + + def test_quality_report_none_default(self): + # Should not raise when quality_report omitted + report = build_certificate_report(_cert()) + assert isinstance(report, str) + + +# --------------------------------------------------------------------------- +# Group 7: WrapAndClaimDiscipline (13 tests) +# --------------------------------------------------------------------------- + +class TestWrapAndClaimDiscipline: + """_wrap() utility and claim-discipline in report output.""" + + def test_wrap_returns_list(self): + result = _wrap("hello world") + assert isinstance(result, list) + + def test_wrap_short_text_single_line(self): + result = _wrap("hello", width=40, indent=2) + assert len(result) == 1 + + def test_wrap_long_text_multiple_lines(self): + long_text = "word " * 30 + result = _wrap(long_text, width=40, indent=2) + assert len(result) > 1 + + def test_wrap_preserves_indent(self): + result = _wrap("hello", width=40, indent=4) + assert result[0].startswith(" ") + + def test_wrap_empty_text(self): + result = _wrap("", width=40, indent=2) + assert isinstance(result, list) + + def test_report_no_proven_language(self): + report = build_certificate_report(_cert()) + # Footer says "No biological activity has been confirmed" which is OK + # but report body should not say "proven active" etc. + assert "proven active" not in report.lower() + assert "proven to be" not in report.lower() + + def test_report_no_cure_language(self): + report = build_certificate_report(_cert()) + assert "cure" not in report.lower() + + def test_report_no_drug_language_in_body(self): + report = build_certificate_report(_cert()) + # "drug" might appear in footer context but not as overclaim + assert "drug candidate" not in report.lower() + + def test_report_no_clinical_language_in_body(self): + report = build_certificate_report(_cert()) + # Check body sections before footer + before_notice = report.split("NOTICE")[0] + assert "clinically proven" not in before_notice.lower() + + def test_references_checked_section_label(self): + report = build_certificate_report(_cert()) + assert "REFERENCES CHECKED" in report + + def test_references_value_in_report(self): + cert = _cert() + report = build_certificate_report(cert) + refs = cert.get("references_checked", []) + for ref in refs: + assert ref in report + + def test_empty_references_shows_none(self): + cert = build_certificate(_make_scored(), {}, []) + report = build_certificate_report(cert) + # references_checked is empty → shows (none) + assert "(none)" in report or "REFERENCES CHECKED" in report + + def test_report_line_lengths_reasonable(self): + report = build_certificate_report(_cert()) + lines = report.split("\n") + # Most lines should be reasonable length (separator can be 72) + long_lines = [l for l in lines if len(l) > 90] + # Allow a few long lines (e.g. the caveat text) + assert len(long_lines) < 10 diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 26e73559..b7cd00ff 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 6457 +BASELINE = 6520 TOLERANCE = 0.05 # allow 5% fluctuation