diff --git a/Makefile b/Makefile index 4944297b..6f4adcbb 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -620,3 +620,9 @@ validate-rejection-events: --input examples/rejection_events_example.json \ --out-json outputs/rejection_events_validation.json \ --out-md outputs/rejection_events_validation.md + +negative-result-dashboard: + PYTHONPATH=src $(PYTHON) scripts/negative_result_dashboard.py \ + --input examples/negative_result_dashboard_example.json \ + --out-json outputs/negative_result_dashboard.json \ + --out-md outputs/negative_result_dashboard.md diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 83465a4e..c6c2e6cc 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,9 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase F F8 — bulk rejection-event validator — v0.5.79) +> **Last updated:** 2026-07-09 (Phase F F9 — negative-result dashboard — v0.5.79) +> **New in v0.5.79:** Negative-result dashboard (F9) — CLI (`scripts/negative_result_dashboard.py`) that reads a collection of negative-result entries from a JSON file and produces a structured dashboard with summary statistics (by category, by pipeline version), score distributions (activity, safety, novelty, ensemble), per-category cross-analysis, and pipeline insights (most common failure category, recalibration opportunities). Outputs JSON + Markdown. Example file (`examples/negative_result_dashboard_example.json`) with 15 toy entries across all 6 failure categories. Schema (`schemas/negative_result_dashboard.schema.json`). `make negative-result-dashboard` target. 33 tests. 2883 total. +> > **New in v0.5.79:** Bulk rejection-event validator (F8) — CLI (`scripts/validate_rejection_events.py`) that reads a JSON list of rejection events, validates each `rejection_code` against the rejection taxonomy (`examples/rejection_taxonomy_example.json`), checks required fields (candidate_id, rejection_code, date, pipeline_version), and outputs a PASS/FAIL validation report with per-event errors, rejection-code distribution, and Markdown summary. Example file (`examples/rejection_events_example.json`) with 6 toy events. `make validate-rejection-events` target. 26 tests. 2849 total. > > **New in v0.5.78:** Calibration link from negative-result entries to intake reports (F7) — closes learning loop by tracing each negative-result entry back to its prediction-vs-actual data in the calibration intake report. Schema addition: `intake_report_id` optional field on `negative_result_entry.schema.json`. CLI (`scripts/link_negative_result_to_intake.py`) reads negative-result archive JSON + calibration intake report, links by candidate_id, reports matched/unmatched entries, orphan intake candidates, lab summary for matched entries, and validates intake_report_id references. Produces structured JSON + Markdown link report. 25 tests. 2822 total. @@ -68,7 +70,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.79 +> **Pipeline version:** v0.5.79 (F9) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 0074745e..8a0c11cf 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,59 @@ # Roadmap +## v0.5.79 — Loop 79: Phase F F9 — Negative-Result Dashboard ✓ (2026-07-09) + +CLI that reads a collection of negative-result entries from a JSON file and +produces a structured dashboard with summary statistics, score distributions, +per-category cross-analysis, and pipeline insights. Shows the value of +failures by making failure patterns reviewable at a glance. + +Changes: +- `scripts/negative_result_dashboard.py` (F9) — Standalone CLI that loads + negative-result entries (list or dict with `entries` key), builds a dashboard + with category breakdown, pipeline version distribution, per-score statistics + (mean, min, max, std), per-category score cross-analysis, and insights (most + common failure category, recalibration opportunities). Supports `--out-json` + and `--out-md` output. Exit 0 on success, 2 on input errors. +- `schemas/negative_result_dashboard.schema.json` (F9) — JSON Schema Draft + 2020-12 for the dashboard output. Validates dashboard_metadata, summary + (by_category, by_pipeline_version), scores_distribution (activity, safety, + novelty, ensemble with mean/min/max/std), category_score_summary, insights + (most_common_failure_category, recalibration_opportunities), and _caveat. +- `examples/negative_result_dashboard_example.json` (F9) — Toy example with 15 + entries spanning all 6 failure categories (pre_selection_reject, selected_untested, + lab_inactive, lab_toxic, control_failure, synthesis_failure) across 9 pipeline + versions. Clearly marked EXAMPLE — NOT REAL DATA. +- `Makefile` — Added `negative-result-dashboard` target. +- `tests/evidence/test_negative_result_dashboard.py` — 33 tests covering: + dashboard required structure, summary counts, by-category breakdown, + by-pipeline-version distribution, score distribution stats (min, max, mean, + count), insights (most common category, highest activity category, lowest + safety category, recalibration opportunities), empty entries handling, + entries without scores, category score summary shape, markdown sections + (title, summary, category table, score distribution, insights, caveat), + example file loading, list/dict input formats, missing file error, empty + list error, missing required fields error, CLI exit codes (0, 2), JSON + output writing, markdown output writing, and example file round-trip. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.79 F9 changelog. Test count: 2883. +- `tests/test_test_count_regression.py` — baseline updated to 2883. + +Honest boundaries: +- Dashboard summaries are computational aggregations of pipeline failure + records, not biological measurements. +- A category with many entries may reflect pipeline prioritisation choices + (e.g., more candidates pass through pre-selection than reach lab testing) + rather than intrinsic biological failure rates. +- Score distributions reflect pipeline-predicted scores, not measured + activity, safety, or novelty. +- The "most common failure category" insight is a descriptive count, not + evidence that the pipeline is failing correctly or incorrectly. +- Recalibration opportunity counts track entries flagged with + `recalibration_used=yes`; they do not measure whether recalibration + actually improved predictions. +- The dashboard does not determine whether rejected candidates would have + been biologically active or safe. All conclusions require qualified + human review and wet-lab validation. + ## v0.5.79 — Loop 78: Phase F F8 — Bulk Rejection-Event Validator ✓ (2026-07-09) CLI that validates rejection events against the rejection taxonomy. Checks diff --git a/examples/negative_result_dashboard_example.json b/examples/negative_result_dashboard_example.json new file mode 100644 index 00000000..d462b2b6 --- /dev/null +++ b/examples/negative_result_dashboard_example.json @@ -0,0 +1,252 @@ +{ + "schema": "EXAMPLE — NOT REAL DATA. For demonstration only.", + "description": "Toy negative-result entries for dashboard demonstration. All candidates, sequences, scores, and reasons are synthetic and must not be treated as real experimental results.", + "pipeline_version": "v0.5.79", + "entries": [ + { + "entry_id": 101, + "date": "2026-06-01", + "candidate_id": "DASH_EXAMPLE-001", + "sequence": "RRIRIIRRIRIIRRI", + "reason_category": "pre_selection_reject", + "reason_detail": "Sequence exceeds maximum allowed charge threshold of +6 (net charge +8). Flagged by pre-selection charge filter.", + "pipeline_version": "v0.5.70", + "source_batch": "wave0.5", + "score_activity": 0.82, + "score_safety": 0.35, + "score_novelty": 0.55, + "score_ensemble": 0.72, + "recalibration_used": "no" + }, + { + "entry_id": 102, + "date": "2026-06-03", + "candidate_id": "DASH_EXAMPLE-002", + "sequence": "GLLRRLRDFLKKLG", + "reason_category": "pre_selection_reject", + "reason_detail": "Patent near-match flagged (Tanimoto similarity 0.89 to patent WO2020XXXXX). Referred for IP review.", + "pipeline_version": "v0.5.71", + "source_batch": "wave0.5", + "score_activity": 0.79, + "score_safety": 0.72, + "score_novelty": 0.31, + "score_ensemble": 0.68, + "recalibration_used": "no" + }, + { + "entry_id": 103, + "date": "2026-06-05", + "candidate_id": "DASH_EXAMPLE-003", + "sequence": "KLAKLAKKLAKLAK", + "reason_category": "selected_untested", + "reason_detail": "Candidate selected for Wave 0.5 panel but deprioritised due to budget constraints. Remains in pool for future waves.", + "pipeline_version": "v0.5.72", + "source_batch": "wave0.5", + "score_activity": 0.85, + "score_safety": 0.90, + "score_novelty": 0.48, + "score_ensemble": 0.79, + "recalibration_used": "no" + }, + { + "entry_id": 104, + "date": "2026-06-10", + "candidate_id": "DASH_EXAMPLE-004", + "sequence": "RWQWRWQWRWQWRW", + "reason_category": "lab_inactive", + "reason_detail": "MIC > 128 ug/mL against E. coli, S. aureus, and P. aeruginosa. Positive control (polymyxin B) showed expected MIC 1-2 ug/mL. Three biological replicates all negative.", + "pipeline_version": "v0.5.73", + "source_batch": "wave0.5", + "assay_type": "MIC", + "assay_result": ">128", + "assay_unit": "ug/mL", + "score_activity": 0.76, + "score_safety": 0.88, + "score_novelty": 0.42, + "score_ensemble": 0.71, + "recalibration_used": "no" + }, + { + "entry_id": 105, + "date": "2026-06-12", + "candidate_id": "DASH_EXAMPLE-005", + "sequence": "GIGKFLHSAKKWGKAFVGEIMNS", + "reason_category": "lab_inactive", + "reason_detail": "MIC > 256 ug/mL against all tested strains. Magainin-2 analog with single-point mutations expected to retain activity but showed complete loss. Structural analysis pending.", + "pipeline_version": "v0.5.73", + "source_batch": "wave0.5", + "assay_type": "MIC", + "assay_result": ">256", + "assay_unit": "ug/mL", + "score_activity": 0.81, + "score_safety": 0.85, + "score_novelty": 0.39, + "score_ensemble": 0.74, + "recalibration_used": "no" + }, + { + "entry_id": 106, + "date": "2026-06-15", + "candidate_id": "DASH_EXAMPLE-006", + "sequence": "FLPIVAGKLLKGL", + "reason_category": "lab_toxic", + "reason_detail": "HC50 = 8 ug/mL (hemolytic). Hemolysis > 50% at 25 ug/mL. Flagged as unsafe for systemic administration.", + "pipeline_version": "v0.5.74", + "source_batch": "wave0.5", + "assay_type": "hemolysis_RBC", + "assay_result": "8", + "assay_unit": "ug/mL HC50", + "score_activity": 0.88, + "score_safety": 0.42, + "score_novelty": 0.52, + "score_ensemble": 0.76, + "recalibration_used": "yes" + }, + { + "entry_id": 107, + "date": "2026-06-18", + "candidate_id": "DASH_EXAMPLE-007", + "sequence": "KWRRWRRWKWRRWR", + "reason_category": "lab_toxic", + "reason_detail": "Cytotoxicity CC50 = 4 ug/mL against HEK293 cells. Selectivity index < 1.0. High membrane disruption activity confirmed.", + "pipeline_version": "v0.5.74", + "source_batch": "wave0.5", + "assay_type": "cytotoxicity_mammalian", + "assay_result": "4", + "assay_unit": "ug/mL CC50", + "score_activity": 0.91, + "score_safety": 0.28, + "score_novelty": 0.58, + "score_ensemble": 0.78, + "recalibration_used": "yes" + }, + { + "entry_id": 108, + "date": "2026-06-20", + "candidate_id": "DASH_EXAMPLE-008", + "sequence": "AAAAKAAAAKAAAA", + "reason_category": "control_failure", + "reason_detail": "Positive control (polymyxin B) MIC = 32 ug/mL against E. coli (expected 1-2 ug/mL). Assay batch invalidated. Candidate not tested.", + "pipeline_version": "v0.5.74", + "source_batch": "wave0.5", + "assay_type": "MIC", + "score_activity": 0.45, + "score_safety": 0.95, + "score_novelty": 0.22, + "score_ensemble": 0.50, + "recalibration_used": "no" + }, + { + "entry_id": 109, + "date": "2026-06-22", + "candidate_id": "DASH_EXAMPLE-009", + "sequence": "RRWQWRMKKLG", + "reason_category": "synthesis_failure", + "reason_detail": "SPPS purification yield < 5%. Multiple hydrophobic stretches caused aggregation during synthesis. Three synthesis attempts all failed.", + "pipeline_version": "v0.5.75", + "source_batch": "wave0.5", + "score_activity": 0.73, + "score_safety": 0.80, + "score_novelty": 0.61, + "score_ensemble": 0.70, + "recalibration_used": "no" + }, + { + "entry_id": 110, + "date": "2026-06-25", + "candidate_id": "DASH_EXAMPLE-010", + "sequence": "ACDEFGHIKLMNPQRSTVWY", + "reason_category": "synthesis_failure", + "reason_detail": "Trp oxidation detected during cleavage. Crude purity < 60%. Not suitable for biological testing without significant re-optimization.", + "pipeline_version": "v0.5.75", + "source_batch": "wave0.5", + "score_activity": 0.68, + "score_safety": 0.75, + "score_novelty": 0.45, + "score_ensemble": 0.65, + "recalibration_used": "no" + }, + { + "entry_id": 111, + "date": "2026-06-28", + "candidate_id": "DASH_EXAMPLE-011", + "sequence": "KLLKLLLKLLKLLL", + "reason_category": "selected_untested", + "reason_detail": "Candidate selected for Wave 0.5b but deferred pending safety assay optimisation. Expected to be tested in Wave 0.5c.", + "pipeline_version": "v0.5.76", + "source_batch": "wave0.5b", + "score_activity": 0.83, + "score_safety": 0.87, + "score_novelty": 0.35, + "score_ensemble": 0.75, + "recalibration_used": "no" + }, + { + "entry_id": 112, + "date": "2026-07-01", + "candidate_id": "DASH_EXAMPLE-012", + "sequence": "RRWRRWRRWRRW", + "reason_category": "lab_inactive", + "reason_detail": "No bactericidal activity at 64 ug/mL against any of 5 tested strains. Time-kill assay confirmed static effect only at highest concentration.", + "pipeline_version": "v0.5.77", + "source_batch": "wave0.5b", + "assay_type": "MIC", + "assay_result": ">64", + "assay_unit": "ug/mL", + "score_activity": 0.80, + "score_safety": 0.52, + "score_novelty": 0.60, + "score_ensemble": 0.73, + "recalibration_used": "no" + }, + { + "entry_id": 113, + "date": "2026-07-03", + "candidate_id": "DASH_EXAMPLE-013", + "sequence": "GIGAVILKVATGLPTLISWIK", + "reason_category": "pre_selection_reject", + "reason_detail": "Known toxic sequence pattern detected (melittin-like). Flagged by safety pre-filter.", + "pipeline_version": "v0.5.77", + "source_batch": "wave0.5b", + "score_activity": 0.87, + "score_safety": 0.15, + "score_novelty": 0.33, + "score_ensemble": 0.67, + "recalibration_used": "no" + }, + { + "entry_id": 114, + "date": "2026-07-05", + "candidate_id": "DASH_EXAMPLE-014", + "sequence": "LLRLLRRLLRLLRR", + "reason_category": "control_failure", + "reason_detail": "Hemolysis control (Triton X-100) outside expected range. Assay batch flagged for QC review. Results not valid.", + "pipeline_version": "v0.5.78", + "source_batch": "wave0.5b", + "assay_type": "hemolysis_RBC", + "score_activity": 0.55, + "score_safety": 0.70, + "score_novelty": 0.30, + "score_ensemble": 0.52, + "recalibration_used": "no" + }, + { + "entry_id": 115, + "date": "2026-07-07", + "candidate_id": "DASH_EXAMPLE-015", + "sequence": "RWKWRWKWRWKW", + "reason_category": "lab_inactive", + "reason_detail": "Biofilm inhibition < 25% at 128 ug/mL against S. aureus biofilm. Anti-biofilm assay showed no significant activity compared to untreated control.", + "pipeline_version": "v0.5.78", + "source_batch": "wave0.5b", + "assay_type": "biofilm_inhibition", + "assay_result": "<25", + "assay_unit": "% inhibition at 128 ug/mL", + "score_activity": 0.77, + "score_safety": 0.65, + "score_novelty": 0.55, + "score_ensemble": 0.71, + "recalibration_used": "no" + } + ] +} diff --git a/schemas/negative_result_dashboard.schema.json b/schemas/negative_result_dashboard.schema.json new file mode 100644 index 00000000..f7a2a1fc --- /dev/null +++ b/schemas/negative_result_dashboard.schema.json @@ -0,0 +1,92 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Negative-Result Dashboard", + "description": "Schema for the negative-result dashboard output. Produced by scripts/negative_result_dashboard.py from a collection of negative-result entries. Dashboard is informational only — it summarises failure patterns, not biological truth.", + "type": "object", + "required": [ + "dashboard_metadata", + "summary", + "scores_distribution", + "category_score_summary", + "insights", + "_caveat" + ], + "properties": { + "dashboard_metadata": { + "type": "object", + "required": ["generated_at", "pipeline_version", "source_file", "total_entries", "pipeline_version_count"], + "properties": { + "generated_at": {"type": "string", "description": "ISO 8601 timestamp of dashboard generation"}, + "pipeline_version": {"type": "string", "description": "Pipeline version used to generate dashboard"}, + "source_file": {"type": "string", "description": "Path to the source negative-result entries file"}, + "total_entries": {"type": "integer", "minimum": 0}, + "pipeline_version_count": {"type": "integer", "minimum": 0} + } + }, + "summary": { + "type": "object", + "required": ["total_entries", "by_category", "by_pipeline_version"], + "properties": { + "total_entries": {"type": "integer", "minimum": 0}, + "by_category": { + "type": "object", + "description": "Count of entries per reason_category", + "additionalProperties": {"type": "integer", "minimum": 0} + }, + "by_pipeline_version": { + "type": "object", + "description": "Count of entries per pipeline_version", + "additionalProperties": {"type": "integer", "minimum": 0} + } + } + }, + "scores_distribution": { + "type": "object", + "description": "Aggregate score statistics across all entries that have score fields", + "required": ["activity", "safety", "novelty", "ensemble"], + "properties": { + "activity": {"$ref": "#/$defs/scoreStats"}, + "safety": {"$ref": "#/$defs/scoreStats"}, + "novelty": {"$ref": "#/$defs/scoreStats"}, + "ensemble": {"$ref": "#/$defs/scoreStats"} + } + }, + "category_score_summary": { + "type": "object", + "description": "Per-category mean scores for cross-analysis" + }, + "insights": { + "type": "object", + "required": [ + "total_failures_analyzed", + "most_common_failure_category", + "most_common_failure_count", + "category_with_highest_avg_activity", + "category_with_lowest_avg_safety" + ], + "properties": { + "total_failures_analyzed": {"type": "integer"}, + "most_common_failure_category": {"type": "string"}, + "most_common_failure_count": {"type": "integer"}, + "category_with_highest_avg_activity": {"type": "string"}, + "category_with_lowest_avg_safety": {"type": "string"}, + "recalibration_opportunities": {"type": "integer"}, + "note": {"type": "string"} + } + }, + "_caveat": {"type": "string"} + }, + "$defs": { + "scoreStats": { + "type": "object", + "required": ["mean", "min", "max", "std", "count"], + "properties": { + "mean": {"type": "number"}, + "min": {"type": "number"}, + "max": {"type": "number"}, + "std": {"type": "number"}, + "count": {"type": "integer"} + } + } + } +} diff --git a/scripts/negative_result_dashboard.py b/scripts/negative_result_dashboard.py new file mode 100644 index 00000000..87a4574a --- /dev/null +++ b/scripts/negative_result_dashboard.py @@ -0,0 +1,319 @@ +"""Generate a negative-result dashboard from a collection of negative-result entries. + +Reads a JSON file containing negative-result entries (see +examples/negative_result_dashboard_example.json) and produces a structured +dashboard with summary statistics, score distributions, per-category analysis, +and pipeline insights. Outputs JSON and/or Markdown. + +Usage: + python scripts/negative_result_dashboard.py \\ + --input examples/negative_result_dashboard_example.json \\ + --out-json outputs/negative_result_dashboard.json \\ + --out-md outputs/negative_result_dashboard.md + +Exit codes: + 0 Success + 2 Input error (missing file, invalid JSON, wrong structure) +""" + +from __future__ import annotations + +import json +import math +import sys +from datetime import datetime, timezone +from pathlib import Path + +REQUIRED_ENTRY_FIELDS = {"entry_id", "candidate_id", "reason_category"} +SCORE_FIELDS = ["score_activity", "score_safety", "score_novelty", "score_ensemble"] +SHORT_SCORE_NAMES = {"score_activity": "activity", "score_safety": "safety", "score_novelty": "novelty", "score_ensemble": "ensemble"} + + +def _stats(values: list[float]) -> dict: + n = len(values) + if n == 0: + return {"mean": 0.0, "min": 0.0, "max": 0.0, "std": 0.0, "count": 0} + mean = sum(values) / n + variance = sum((v - mean) ** 2 for v in values) / n + return { + "mean": round(mean, 4), + "min": round(min(values), 4), + "max": round(max(values), 4), + "std": round(math.sqrt(variance), 4), + "count": n, + } + + +def _count_by(entries: list[dict], key: str) -> dict[str, int]: + counts: dict[str, int] = {} + for e in entries: + val = e.get(key, "unknown") + counts[val] = counts.get(val, 0) + 1 + return dict(sorted(counts.items())) + + +def build_dashboard(entries: list[dict], source_file: str = "") -> dict: + total = len(entries) + + by_category = _count_by(entries, "reason_category") + by_pipeline_version = _count_by(entries, "pipeline_version") + + score_values: dict[str, list[float]] = {name: [] for name in SCORE_FIELDS} + for e in entries: + for field in SCORE_FIELDS: + val = e.get(field) + if isinstance(val, (int, float)): + score_values[field].append(float(val)) + + scores_distribution = {} + for field, vals in score_values.items(): + short = SHORT_SCORE_NAMES.get(field, field) + scores_distribution[short] = _stats(vals) + + category_scores: dict[str, dict[str, list[float]]] = {} + for e in entries: + cat = e.get("reason_category", "unknown") + if cat not in category_scores: + category_scores[cat] = {name: [] for name in SCORE_FIELDS} + for field in SCORE_FIELDS: + val = e.get(field) + if isinstance(val, (int, float)): + category_scores[cat][field].append(float(val)) + + category_score_summary = {} + for cat, fields in category_scores.items(): + category_score_summary[cat] = {} + for field, vals in fields.items(): + short = SHORT_SCORE_NAMES.get(field, field) + category_score_summary[cat][short] = _stats(vals) + + if by_category: + most_common_cat = max(by_category, key=by_category.get) + most_common_count = by_category[most_common_cat] + else: + most_common_cat = "none" + most_common_count = 0 + + cat_mean_activity = {} + cat_mean_safety = {} + for cat, entry_stats in category_score_summary.items(): + if "activity" in entry_stats: + cat_mean_activity[cat] = entry_stats["activity"]["mean"] + if "safety" in entry_stats: + cat_mean_safety[cat] = entry_stats["safety"]["mean"] + + highest_activity_cat = max(cat_mean_activity, key=cat_mean_activity.get) if cat_mean_activity else "none" + lowest_safety_cat = min(cat_mean_safety, key=cat_mean_safety.get) if cat_mean_safety else "none" + + recalibration_count = sum( + 1 for e in entries if e.get("recalibration_used") == "yes" + ) + + return { + "dashboard_metadata": { + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "pipeline_version": "v0.5.79", + "source_file": source_file, + "total_entries": total, + "pipeline_version_count": len(by_pipeline_version), + }, + "summary": { + "total_entries": total, + "by_category": by_category, + "by_pipeline_version": by_pipeline_version, + }, + "scores_distribution": scores_distribution, + "category_score_summary": category_score_summary, + "insights": { + "total_failures_analyzed": total, + "most_common_failure_category": most_common_cat, + "most_common_failure_count": most_common_count, + "category_with_highest_avg_activity": highest_activity_cat, + "category_with_lowest_avg_safety": lowest_safety_cat, + "recalibration_opportunities": recalibration_count, + "note": ( + "Dashboard insights are computational summaries of pipeline " + "failure records. They are not biological proof. A high-scoring " + "failure category does not mean the pipeline is failing correctly " + "or incorrectly — it means that category generates more entries. " + "Score distributions reflect pipeline-predicted scores, not " + "measured activity or safety." + ), + }, + "_caveat": ( + "This dashboard is informational only. It summarises the structure " + "and distribution of negative-result entries in the archive. " + "A category with many entries may reflect pipeline prioritisation " + "choices, not biological failure rate. Score distributions are " + "computational predictions and must not be treated as measured " + "values. This dashboard does not determine whether rejected " + "candidates would have been biologically active or safe. " + "All conclusions about candidate quality require qualified " + "human review and wet-lab validation." + ), + } + + +def generate_markdown(dashboard: dict) -> str: + lines: list[str] = [] + lines.append("# Negative-Result Dashboard\n") + meta = dashboard.get("dashboard_metadata", {}) + lines.append(f"**Generated:** {meta.get('generated_at', '?')} ") + lines.append(f"**Pipeline version:** {meta.get('pipeline_version', '?')} ") + lines.append(f"**Total entries analyzed:** {meta.get('total_entries', 0)}\n") + + summary = dashboard.get("summary", {}) + total = summary.get("total_entries", 0) + lines.append("## Summary\n") + lines.append(f"**Total negative-result entries:** {total}\n") + + by_category = summary.get("by_category", {}) + if by_category: + lines.append("### By Failure Category\n") + lines.append("| Category | Count | Percentage |") + lines.append("|----------|-------|------------|") + for cat, count in by_category.items(): + pct = f"{100 * count / total:.1f}%" if total > 0 else "0%" + lines.append(f"| {cat} | {count} | {pct} |") + lines.append("") + + by_pv = summary.get("by_pipeline_version", {}) + if by_pv: + lines.append("### By Pipeline Version\n") + lines.append("| Pipeline Version | Count |") + lines.append("|------------------|-------|") + for pv, count in by_pv.items(): + lines.append(f"| {pv} | {count} |") + lines.append("") + + sd = dashboard.get("scores_distribution", {}) + if sd: + lines.append("## Score Distribution (all entries)\n") + lines.append("| Score | Mean | Min | Max | Std | Count |") + lines.append("|-------|------|-----|-----|-----|-------|") + for score_name in ["activity", "safety", "novelty", "ensemble"]: + s = sd.get(score_name, {}) + lines.append( + f"| {score_name} | {s.get('mean', '?'):} | " + f"{s.get('min', '?'):} | {s.get('max', '?'):} | " + f"{s.get('std', '?'):} | {s.get('count', 0)} |" + ) + lines.append("") + + css = dashboard.get("category_score_summary", {}) + if css: + lines.append("## Per-Category Score Analysis\n") + for cat, scores in css.items(): + lines.append(f"### {cat}\n") + lines.append("| Score | Mean | Min | Max | Std | Count |") + lines.append("|-------|------|-----|-----|-----|-------|") + for score_name in ["activity", "safety", "novelty", "ensemble"]: + s = scores.get(score_name, {}) + if s.get("count", 0) > 0: + lines.append( + f"| {score_name} | {s.get('mean', '?'):} | " + f"{s.get('min', '?'):} | {s.get('max', '?'):} | " + f"{s.get('std', '?'):} | {s.get('count', 0)} |" + ) + lines.append("") + + insights = dashboard.get("insights", {}) + if insights: + lines.append("## Insights\n") + lines.append( + f"- **Most common failure category:** " + f"{insights.get('most_common_failure_category', '?')} " + f"({insights.get('most_common_failure_count', 0)} entries)" + ) + lines.append( + f"- **Category with highest avg activity score:** " + f"{insights.get('category_with_highest_avg_activity', '?')}" + ) + lines.append( + f"- **Category with lowest avg safety score:** " + f"{insights.get('category_with_lowest_avg_safety', '?')}" + ) + lines.append( + f"- **Recalibration opportunities:** " + f"{insights.get('recalibration_opportunities', 0)} " + f"entries contributed to recalibration" + ) + lines.append( + f"- **Note:** {insights.get('note', '')}" + ) + + lines.append(f"\n## Caveat\n\n{dashboard.get('_caveat', '')}\n") + return "\n".join(lines) + + +def load_entries(input_path: str | Path) -> list[dict]: + path = Path(input_path) + if not path.exists(): + print(f"Error: Input not found: {input_path}", file=sys.stderr) + sys.exit(2) + + try: + data = json.loads(path.read_text()) + except Exception as e: + print(f"Error reading input: {e}", file=sys.stderr) + sys.exit(2) + + if isinstance(data, list): + entries = data + elif isinstance(data, dict): + entries = data.get("entries", data.get("candidates", [])) + if not entries and all(k in REQUIRED_ENTRY_FIELDS for k in data if k != "schema"): + entries = [data] + else: + print("Error: Input must be a JSON array or object with an 'entries' key", file=sys.stderr) + sys.exit(2) + + if not isinstance(entries, list) or not entries: + print("Error: No entries found in input", file=sys.stderr) + sys.exit(2) + + for i, entry in enumerate(entries): + missing = REQUIRED_ENTRY_FIELDS - set(entry.keys()) + if missing: + print( + f"Error: Entry {i} (candidate: {entry.get('candidate_id', '?')}) " + f"missing required fields: {missing}", + file=sys.stderr, + ) + sys.exit(2) + + return entries + + +def main() -> int: + import argparse + + parser = argparse.ArgumentParser( + description="Generate a negative-result dashboard from a collection of entries." + ) + parser.add_argument("--input", required=True, help="Path to negative-result entries JSON file") + parser.add_argument("--out-json", default=None, help="Output JSON dashboard path") + parser.add_argument("--out-md", default=None, help="Output Markdown dashboard path") + + args = parser.parse_args() + + entries = load_entries(args.input) + + dashboard = build_dashboard(entries, source_file=str(args.input)) + + print(json.dumps(dashboard, indent=2)) + + if args.out_json: + Path(args.out_json).write_text(json.dumps(dashboard, indent=2) + "\n") + print(f"\nJSON dashboard written to: {args.out_json}") + + if args.out_md: + md = generate_markdown(dashboard) + Path(args.out_md).write_text(md + "\n") + print(f"Markdown dashboard written to: {args.out_md}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/evidence/test_negative_result_dashboard.py b/tests/evidence/test_negative_result_dashboard.py new file mode 100644 index 00000000..309c294f --- /dev/null +++ b/tests/evidence/test_negative_result_dashboard.py @@ -0,0 +1,379 @@ +"""Tests for the negative-result dashboard generator.""" +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.negative_result_dashboard import ( + build_dashboard, + generate_markdown, + load_entries, +) + +_EXAMPLE_PATH = ( + Path(__file__).resolve().parents[2] / "examples" / "negative_result_dashboard_example.json" +) + + +def _sample_entries() -> list[dict]: + return [ + { + "entry_id": 1, + "candidate_id": "TEST-001", + "sequence": "RRIRIIRRIRIIRRI", + "reason_category": "lab_inactive", + "reason_detail": "MIC > 128 ug/mL.", + "pipeline_version": "v0.5.70", + "source_batch": "test", + "score_activity": 0.80, + "score_safety": 0.85, + "score_novelty": 0.50, + "score_ensemble": 0.75, + "recalibration_used": "no", + }, + { + "entry_id": 2, + "candidate_id": "TEST-002", + "sequence": "KLAKLAKKLAKLAK", + "reason_category": "lab_toxic", + "reason_detail": "HC50 = 8 ug/mL.", + "pipeline_version": "v0.5.71", + "source_batch": "test", + "score_activity": 0.90, + "score_safety": 0.30, + "score_novelty": 0.60, + "score_ensemble": 0.78, + "recalibration_used": "yes", + }, + { + "entry_id": 3, + "candidate_id": "TEST-003", + "sequence": "GIGKFLHSAKKWGKAFVGEIMNS", + "reason_category": "lab_inactive", + "reason_detail": "MIC > 256 ug/mL.", + "pipeline_version": "v0.5.70", + "source_batch": "test", + "score_activity": 0.75, + "score_safety": 0.90, + "score_novelty": 0.40, + "score_ensemble": 0.70, + "recalibration_used": "no", + }, + { + "entry_id": 4, + "candidate_id": "TEST-004", + "sequence": "AAAAKAAAAKAAAA", + "reason_category": "control_failure", + "reason_detail": "Positive control outside range.", + "pipeline_version": "v0.5.72", + "source_batch": "test", + "score_activity": 0.45, + "score_safety": 0.95, + "score_novelty": 0.30, + "score_ensemble": 0.50, + "recalibration_used": "no", + }, + { + "entry_id": 5, + "candidate_id": "TEST-005", + "sequence": "RRWQWRMKKLG", + "reason_category": "synthesis_failure", + "reason_detail": "Yield < 5%.", + "pipeline_version": "v0.5.73", + "source_batch": "test", + "score_activity": 0.70, + "score_safety": 0.80, + "score_novelty": 0.55, + "score_ensemble": 0.68, + "recalibration_used": "no", + }, + ] + + +class TestBuildDashboard: + def test_returns_required_structure(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + assert "dashboard_metadata" in dashboard + assert "summary" in dashboard + assert "scores_distribution" in dashboard + assert "category_score_summary" in dashboard + assert "insights" in dashboard + assert "_caveat" in dashboard + + def test_summary_counts(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + assert dashboard["summary"]["total_entries"] == 5 + + def test_by_category_breakdown(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + by_cat = dashboard["summary"]["by_category"] + assert by_cat["lab_inactive"] == 2 + assert by_cat["lab_toxic"] == 1 + assert by_cat["control_failure"] == 1 + assert by_cat["synthesis_failure"] == 1 + + def test_by_pipeline_version(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + by_pv = dashboard["summary"]["by_pipeline_version"] + assert by_pv["v0.5.70"] == 2 + assert by_pv["v0.5.71"] == 1 + assert by_pv["v0.5.72"] == 1 + assert by_pv["v0.5.73"] == 1 + + def test_scores_distribution_stats(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + sd = dashboard["scores_distribution"] + assert "activity" in sd + assert "safety" in sd + assert "novelty" in sd + assert "ensemble" in sd + assert sd["activity"]["count"] == 5 + assert sd["activity"]["min"] == 0.45 + assert sd["activity"]["max"] == 0.90 + assert round(sd["activity"]["mean"], 2) == 0.72 + + def test_insights_identifies_most_common_category(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + assert dashboard["insights"]["most_common_failure_category"] == "lab_inactive" + assert dashboard["insights"]["most_common_failure_count"] == 2 + + def test_insights_identifies_highest_activity_category(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + assert dashboard["insights"]["category_with_highest_avg_activity"] == "lab_toxic" + + def test_insights_identifies_lowest_safety_category(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + assert dashboard["insights"]["category_with_lowest_avg_safety"] == "lab_toxic" + + def test_insights_recalibration_opportunities(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + assert dashboard["insights"]["recalibration_opportunities"] == 1 + + def test_empty_entries_produces_zero_summary(self): + dashboard = build_dashboard([]) + assert dashboard["summary"]["total_entries"] == 0 + assert dashboard["summary"]["by_category"] == {} + assert dashboard["insights"]["total_failures_analyzed"] == 0 + + def test_entries_without_scores_handled_gracefully(self): + entries = [ + { + "entry_id": 99, + "candidate_id": "NO-SCORE-001", + "reason_category": "pre_selection_reject", + "reason_detail": "No scores available.", + "pipeline_version": "v0.5.70", + "source_batch": "test", + } + ] + dashboard = build_dashboard(entries) + assert dashboard["summary"]["total_entries"] == 1 + assert dashboard["scores_distribution"]["activity"]["count"] == 0 + + def test_category_score_summary_shape(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + css = dashboard["category_score_summary"] + assert "lab_inactive" in css + assert "lab_toxic" in css + assert "activity" in css["lab_inactive"] + assert "safety" in css["lab_inactive"] + assert css["lab_inactive"]["activity"]["count"] == 2 + + def test_metadata_contains_generation_timestamp(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + assert "generated_at" in dashboard["dashboard_metadata"] + assert dashboard["dashboard_metadata"]["total_entries"] == 5 + + +class TestGenerateMarkdown: + def test_contains_title(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + md = generate_markdown(dashboard) + assert "# Negative-Result Dashboard" in md + + def test_contains_summary_section(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + md = generate_markdown(dashboard) + assert "## Summary" in md + + def test_contains_category_table(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + md = generate_markdown(dashboard) + assert "### By Failure Category" in md + assert "lab_inactive" in md + assert "lab_toxic" in md + + def test_contains_score_distribution(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + md = generate_markdown(dashboard) + assert "## Score Distribution" in md + assert "activity" in md + assert "safety" in md + + def test_contains_insights(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + md = generate_markdown(dashboard) + assert "## Insights" in md + assert "Most common failure category" in md + + def test_contains_caveat(self): + entries = _sample_entries() + dashboard = build_dashboard(entries) + md = generate_markdown(dashboard) + assert "## Caveat" in md + + def test_empty_entries_handled(self): + dashboard = build_dashboard([]) + md = generate_markdown(dashboard) + assert "# Negative-Result Dashboard" in md + + +class TestLoadEntries: + def test_loads_from_example_file(self): + entries = load_entries(_EXAMPLE_PATH) + assert len(entries) == 15 + assert entries[0]["entry_id"] == 101 + assert entries[0]["reason_category"] == "pre_selection_reject" + + def test_loads_list_format(self, tmp_path): + data = [ + {"entry_id": 1, "candidate_id": "T1", "reason_category": "lab_inactive"}, + {"entry_id": 2, "candidate_id": "T2", "reason_category": "lab_toxic"}, + ] + inp = tmp_path / "entries.json" + inp.write_text(json.dumps(data)) + entries = load_entries(inp) + assert len(entries) == 2 + + def test_loads_dict_with_entries_key(self, tmp_path): + data = {"entries": [ + {"entry_id": 1, "candidate_id": "T1", "reason_category": "lab_inactive"}, + ]} + inp = tmp_path / "entries.json" + inp.write_text(json.dumps(data)) + entries = load_entries(inp) + assert len(entries) == 1 + + def test_exits_on_missing_file(self, tmp_path): + with pytest.raises(SystemExit) as exc: + load_entries(tmp_path / "nonexistent.json") + assert exc.value.code == 2 + + def test_exits_on_empty_list(self, tmp_path): + inp = tmp_path / "empty.json" + inp.write_text("[]") + with pytest.raises(SystemExit) as exc: + load_entries(inp) + assert exc.value.code == 2 + + def test_exits_on_missing_required_fields(self, tmp_path): + data = [{"entry_id": 1}] # missing candidate_id and reason_category + inp = tmp_path / "bad.json" + inp.write_text(json.dumps(data)) + with pytest.raises(SystemExit) as exc: + load_entries(inp) + assert exc.value.code == 2 + + +class TestCLI: + def test_exit_zero_on_success(self, tmp_path): + entries = _sample_entries() + inp = tmp_path / "entries.json" + inp.write_text(json.dumps(entries)) + r = subprocess.run( + [sys.executable, "scripts/negative_result_dashboard.py", + "--input", str(inp)], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert r.returncode == 0 + + def test_exit_two_on_missing_input(self, tmp_path): + r = subprocess.run( + [sys.executable, "scripts/negative_result_dashboard.py", + "--input", str(tmp_path / "nonexistent.json")], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert r.returncode == 2 + + def test_exit_two_on_empty_list(self, tmp_path): + inp = tmp_path / "empty.json" + inp.write_text("[]") + r = subprocess.run( + [sys.executable, "scripts/negative_result_dashboard.py", + "--input", str(inp)], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert r.returncode == 2 + + def test_exit_two_on_invalid_json(self, tmp_path): + inp = tmp_path / "bad.json" + inp.write_text("not json") + r = subprocess.run( + [sys.executable, "scripts/negative_result_dashboard.py", + "--input", str(inp)], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert r.returncode == 2 + + def test_writes_json_output(self, tmp_path): + entries = _sample_entries() + inp = tmp_path / "entries.json" + inp.write_text(json.dumps(entries)) + out = tmp_path / "dashboard.json" + subprocess.run( + [sys.executable, "scripts/negative_result_dashboard.py", + "--input", str(inp), "--out-json", str(out)], + capture_output=True, + env={"PYTHONPATH": "src"}, + ) + assert out.exists() + data = json.loads(out.read_text()) + assert data["summary"]["total_entries"] == 5 + + def test_writes_markdown_output(self, tmp_path): + entries = _sample_entries() + inp = tmp_path / "entries.json" + inp.write_text(json.dumps(entries)) + out = tmp_path / "dashboard.md" + subprocess.run( + [sys.executable, "scripts/negative_result_dashboard.py", + "--input", str(inp), "--out-md", str(out)], + capture_output=True, + env={"PYTHONPATH": "src"}, + ) + assert out.exists() + assert "# Negative-Result Dashboard" in out.read_text() + + def test_runs_on_example_file(self, tmp_path): + r = subprocess.run( + [sys.executable, "scripts/negative_result_dashboard.py", + "--input", str(_EXAMPLE_PATH)], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + ) + assert r.returncode == 0 + assert "dashboard_metadata" in r.stdout diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 9ae6be9e..ada49412 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 2849 tests as of Loop 78 (2026-07-09) -- Phase F F8 (bulk rejection-event validator) +# Baseline: 2883 tests as of Loop 79 (2026-07-09) -- Phase F F9 (negative-result dashboard) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 2849 +BASELINE = 2883 TOLERANCE = 0.05 # allow 5% fluctuation