diff --git a/docs/METRICS_CURRENT.md b/docs/METRICS_CURRENT.md index 4f2a8b23..5a485056 100644 --- a/docs/METRICS_CURRENT.md +++ b/docs/METRICS_CURRENT.md @@ -161,7 +161,9 @@ representative of diverse AMP classes not covered by the helic-centric scorer. AntiCP 2.0 detects anticancer peptide (ACP) patterns, not antimicrobial activity directly. Mitigation: Wave 0.5b designs avoid aromatic residues and pure amphipathic helix. -Machine-readable: `outputs/wave0_5_external_predict_results.csv`, `outputs/wave0_5_external_consensus.csv` +Current-state summary is documented here. Wave 0.5 machine-readable CSV outputs are +generated locally via `make wave0-5-fill-external` when `outputs/wave05_combined_consensus.csv` +is present; they are not guaranteed to be committed in every checkout. --- diff --git a/docs/WAVE_0_5_EXTERNAL_PREDICTOR_SUMMARY.md b/docs/WAVE_0_5_EXTERNAL_PREDICTOR_SUMMARY.md index d056798d..2bb230f8 100644 --- a/docs/WAVE_0_5_EXTERNAL_PREDICTOR_SUMMARY.md +++ b/docs/WAVE_0_5_EXTERNAL_PREDICTOR_SUMMARY.md @@ -7,17 +7,20 @@ > HemoFinder predicts hemolysis risk, NOT antimicrobial activity. Generated: 2026-06-29 -Status: **PENDING — External predictor submissions required** +Updated: 2026-07-02 +Status: **HISTORICAL TEMPLATE ONLY — retained as the pre-submission plan** --- ## Purpose -This document describes the external predictor screen for the Wave 0.5 shortlist -(60 candidates across 10 new seed families). Results are PENDING because these -predictors require manual web submission to published servers. +This document describes the original pre-submission external predictor plan for the +Wave 0.5 shortlist (60 candidates across 10 new seed families). It is preserved as +historical process context, not as the current result state. -The screen must be completed before final Wave 1 panel selection (Phase 7). +The external screen was completed later and the authoritative current-state summary +now lives in [`docs/METRICS_CURRENT.md`](/Users/james/.codex/worktrees/9f2e/openamp-foundry/docs/METRICS_CURRENT.md) +and [`docs/ROADMAP.md`](/Users/james/.codex/worktrees/9f2e/openamp-foundry/docs/ROADMAP.md). --- @@ -108,24 +111,25 @@ confirmation required before synthesis. --- -## How to Fill In +## Historical Submission Procedure 1. Export `outputs/wave0_5_internal_shortlist.csv` → extract sequences column 2. Create FASTA file from all 60 sequences 3. Submit to each tool's web interface (see URLs above) 4. Record Y/N (or probability score) per candidate per tool 5. Fill in `outputs/wave0_5_external_predict_results.csv` -6. Run: `make wave0-5-external-consensus` (Makefile target — to be added in Phase 10) +6. Run `make wave0-5-fill-external` to derive the Wave 0.5 result files when + `outputs/wave05_combined_consensus.csv` is available in the local checkout. --- -## Machine-Readable Status +## Historical Artifact Expectations | File | Status | |---|---| -| `outputs/wave0_5_external_predict_results.csv` | PENDING (60 rows, all PENDING) | -| `outputs/wave0_5_external_consensus.csv` | PENDING (internal OpenAMP scores available) | -| `outputs/wave0_5_safety_consensus.csv` | PENDING (internal estimates in internal_safety_estimate column) | +| `outputs/wave0_5_external_predict_results.csv` | Generated only after running `make wave0-5-fill-external` with local source data | +| `outputs/wave0_5_external_consensus.csv` | Generated only after running `make wave0-5-fill-external` with local source data | +| `outputs/wave0_5_safety_consensus.csv` | Generated only after running `make wave0-5-fill-external` with local source data | --- diff --git a/outputs/metrics_snapshot.json b/outputs/metrics_snapshot.json new file mode 100644 index 00000000..497bc291 --- /dev/null +++ b/outputs/metrics_snapshot.json @@ -0,0 +1,162 @@ +{ + "benchmark_inputs": { + "amp_csv": "examples/validation/known_amps.csv", + "cluster_threshold": 0.7, + "decoy_csv": "examples/validation/random_background.csv", + "hemolysis_csv": "examples/validation/hemolysis_reference.csv", + "n_bootstrap": 2000, + "phase3_config": "configs/phase3.yaml", + "standard_config": "configs/pipeline.yaml" + }, + "cluster_split": { + "cluster_aware_ci95": [ + 0.7061, + 0.8526 + ], + "full_auroc": 0.7832, + "held_out_auroc": 0.8734, + "interpretation": "STRONG — AUROC > 0.70 and cluster-aware CI lower bound > 0.65. Signal survives near-duplicate de-inflation.", + "n_amps_in_multi_member_clusters": 33, + "n_clusters": 76, + "n_multi_member_clusters": 14, + "representative_auroc": 0.7607, + "representative_ci95": [ + 0.6854, + 0.8301 + ], + "standard_ci95": [ + 0.717, + 0.8423 + ] + }, + "expert_ablation": { + "anti_signal_components": [ + "synthesis", + "safety", + "hemolysis_safety", + "serum_stability" + ], + "delta_auroc": -0.0713, + "ensemble_auroc": 0.7832, + "expert_auroc": 0.7119, + "near_zero_components": [ + "hinge_selectivity", + "novelty", + "motif_novelty", + "boman_activity" + ], + "signal_bearing_components": [ + "activity", + "selectivity_proxy" + ], + "verdict": "Expert composite (0.7119) scores LOWER than ensemble (0.7832) by -0.0713 AUROC. The added components degrade discrimination on this benchmark. The expert composite should NOT replace the ensemble for AMP/non-AMP triage without further evidence." + }, + "phase3": { + "auprc": 0.7933, + "auroc": 0.7448, + "interpretation": "STRONG — model has meaningful discriminative power (AUROC > 0.70)", + "n_negatives": 96, + "n_positives": 95, + "n_total": 191 + }, + "schema_version": 1, + "selectivity": { + "expert_composite_verdict": "Expert composite detects hemolysis risk better than ensemble (detection AUROC 0.5459 vs 0.4201). The added selectivity/safety components earn their keep on within-AMP ranking.", + "n_border": 59, + "n_hemolytic": 54, + "n_selective": 125, + "risk_detectors": [ + "serum_stability" + ], + "risk_indicators": [ + "expert_composite", + "safety", + "synthesis", + "boman_activity", + "selectivity_proxy", + "hemolysis_risk" + ], + "safety_verdict": "Safety scorer shows weak hemolysis detection (detection AUROC=0.5116, CI lo=0.4321 < 0.5). Signal is present but not statistically significant. The melittin blind spot remains: 1D hydrophobic moment cannot capture all hemolysis mechanisms.", + "selectivity_proxy_verdict": "Selectivity proxy shows weak hemolysis detection (detection AUROC=0.5744, CI lo=0.4954). Trend is correct but not significant at this sample size." + }, + "standard": { + "auprc": 0.8164, + "auroc": 0.7832, + "interpretation": "STRONG — model has meaningful discriminative power (AUROC > 0.70)", + "n_negatives": 96, + "n_positives": 95, + "n_total": 191, + "recall_at_10": 0.1053, + "recall_at_20": 0.2105, + "recall_at_43": 0.4211 + }, + "triage": { + "best_scorer": "selectivity_proxy", + "per_scorer": { + "activity": { + "hemolytic_vs_decoy": 0.934, + "selective_vs_decoy": 0.8846, + "selective_vs_hemolytic": 0.43, + "triages_correctly": false + }, + "ensemble": { + "hemolytic_vs_decoy": 0.8911, + "selective_vs_decoy": 0.8481, + "selective_vs_hemolytic": 0.4661, + "triages_correctly": false + }, + "hemolysis_risk": { + "hemolytic_vs_decoy": 0.4924, + "selective_vs_decoy": 0.485, + "selective_vs_hemolytic": 0.4876, + "triages_correctly": false + }, + "safe_weighted_ensemble": { + "hemolytic_vs_decoy": 0.8904, + "selective_vs_decoy": 0.849, + "selective_vs_hemolytic": 0.483, + "triages_correctly": false + }, + "safety": { + "hemolytic_vs_decoy": 0.3, + "selective_vs_decoy": 0.344, + "selective_vs_hemolytic": 0.5384, + "triages_correctly": false + }, + "selectivity_proxy": { + "hemolytic_vs_decoy": 0.7955, + "selective_vs_decoy": 0.782, + "selective_vs_hemolytic": 0.6095, + "triages_correctly": true + }, + "serum_stability": { + "hemolytic_vs_decoy": 0.1604, + "selective_vs_decoy": 0.2174, + "selective_vs_hemolytic": 0.5686, + "triages_correctly": false + }, + "synthesis": { + "hemolytic_vs_decoy": 0.6336, + "selective_vs_decoy": 0.5898, + "selective_vs_hemolytic": 0.4692, + "triages_correctly": false + }, + "triage_score": { + "hemolytic_vs_decoy": 0.9019, + "selective_vs_decoy": 0.8629, + "selective_vs_hemolytic": 0.4622, + "triages_correctly": false + } + }, + "top_20_by_ensemble": { + "DECOY": 0, + "HEMOLYTIC": 6, + "SELECTIVE": 14 + }, + "top_20_by_triage_score": { + "DECOY": 0, + "HEMOLYTIC": 4, + "SELECTIVE": 16 + } + } +} diff --git a/tests/test_docs_consistent.py b/tests/test_docs_consistent.py index 11057c14..c96d63d8 100644 --- a/tests/test_docs_consistent.py +++ b/tests/test_docs_consistent.py @@ -2,8 +2,8 @@ from __future__ import annotations -import re import json +import re from pathlib import Path @@ -19,6 +19,11 @@ EXPANDED_PR_REF = "PR #110" ORIGINAL_DEMO_AUROC = "0.8420" SNAPSHOT_PATH = REPO_ROOT / "outputs" / "metrics_snapshot.json" +WAVE05_GENERATED_OUTPUTS = [ + REPO_ROOT / "outputs" / "wave0_5_external_predict_results.csv", + REPO_ROOT / "outputs" / "wave0_5_external_consensus.csv", + REPO_ROOT / "outputs" / "wave0_5_safety_consensus.csv", +] # Patterns that indicate stale "current" usage (not historical context) _STALE_CURRENT_PATTERNS: list[str] = [ @@ -144,3 +149,17 @@ def test_doc_wet_lab_probability_has_honest_correction(self): assert "55–80%" in text or "55-80" in text or "honest" in text.lower(), ( "WET_LAB_PROBABILITY.md missing honest correction range" ) + + def test_doc_wave05_generated_outputs_claimed_only_if_present(self): + """Docs must not present missing Wave 0.5 generated CSVs as committed facts.""" + files_present = all(path.exists() for path in WAVE05_GENERATED_OUTPUTS) + + metrics_text = (DOCS_DIR / "METRICS_CURRENT.md").read_text(encoding="utf-8") + summary_text = (DOCS_DIR / "WAVE_0_5_EXTERNAL_PREDICTOR_SUMMARY.md").read_text( + encoding="utf-8" + ) + + if not files_present: + assert "Machine-readable: `outputs/wave0_5_external_predict_results.csv`" not in metrics_text + assert "PENDING (60 rows, all PENDING)" not in summary_text + assert "Generated only after running `make wave0-5-fill-external`" in summary_text