diff --git a/docs/NOVELTY_BROAD_CHECK.md b/docs/NOVELTY_BROAD_CHECK.md new file mode 100644 index 00000000..9db69ff6 --- /dev/null +++ b/docs/NOVELTY_BROAD_CHECK.md @@ -0,0 +1,83 @@ +# Broad Novelty Check — Pilot Panel vs Curated AMP Database + +> Generated: 2026-06-28T16:35:23.456115+00:00 +> Panel: outputs/pilot_panel.csv +> Reference database: examples/known_reference/amp_curated_references.csv (72 unique sequences) +> Thresholds: KNOWN_VARIANT ≥ 70%, CLOSE_RELATIVE ≥ 50% + +## Purpose + +The standard pipeline novelty score compares candidates against ~45 seed sequences. +This report extends the comparison to the curated AMP reference database to detect +near-copies of published AMPs that the seed-based novelty score may miss. + +## Summary + +| Category | Count | Description | +|---|:---:|---| +| KNOWN_VARIANT | 3 | ≥70% similar to a known published AMP | +| CLOSE_RELATIVE | 1 | 50%–70% similar to a known AMP | +| NOVEL | 16 | <50% similar — no close match in reference database | +| **Total** | **20** | | + +## Per-Candidate Results + +| Rank | Candidate | Sequence | Seed | Ensemble | Seed-Novelty | Broad-Sim | Best-Match | Category | +|--:|---|---|---|:---:|:---:|:---:|---|---| +| 1 | SEED-009_VAR_033 | `RRLPRPGYMPRP` | SEED-009 | 0.807 | 0.692 | 0.308 | REF-TRP-001 (tryptophan_rich_cathelicidin) | ✓ NOVEL | +| 2 | SEED-009_VAR_027 | `RRLPRGPYLPKP` | SEED-009 | 0.808 | 0.692 | 0.308 | REF-TRP-001 (tryptophan_rich_cathelicidin) | ✓ NOVEL | +| 3 | SEED-007_VAR_009 | `IKFTTMLRKLG` | SEED-007 | 0.849 | 0.727 | 0.273 | REF-RRW-001 (tachyplesin_like) | ✓ NOVEL | +| 4 | SEED-007_VAR_001 | `AKITTMLKKLG` | SEED-007 | 0.824 | 0.636 | 0.364 | REF-RRW-001 (tachyplesin_like) | ✓ NOVEL | +| 5 | SEED-007_VAR_018 | `IKISTMLKKAG` | SEED-007 | 0.815 | 0.647 | 0.353 | REF-PIS-001 (piscidin) | ✓ NOVEL | +| 6 | SEED-009_VAR_039 | `RRLPRPPYIPRG` | SEED-009 | 0.796 | 0.650 | 0.350 | REF-PYR-001 (pyrrhocoricin) | ✓ NOVEL | +| 7 | SEED-009_VAR_017 | `RRLGRPPYLGRP` | SEED-009 | 0.798 | 0.647 | 0.353 | REF-BUF-002 (buforin_ii) | ✓ NOVEL | +| 8 | SEED-007_VAR_035 | `IKITTMAKKVG` | SEED-007 | 0.806 | 0.647 | 0.353 | REF-PIS-001 (piscidin) | ✓ NOVEL | +| 9 | SEED-006_VAR_059 | `INWKPIAAMAKKLV` | SEED-006 | 0.821 | 0.643 | 0.357 | REF-RRW-001 (tachyplesin_like) | ✓ NOVEL | +| 10 | SEED-006_VAR_071 | `IQWKGIAAMAKRLL` | SEED-006 | 0.828 | 0.643 | 0.357 | REF-SCO-001 (scorpion_isct) | ✓ NOVEL | +| 11 | SEED-006_VAR_062 | `INWRGIAAMAKKFL` | SEED-006 | 0.841 | 0.643 | 0.357 | REF-LL37-002 (cathelicidin_ll37_fragment) | ✓ NOVEL | +| 12 | SEED-006_VAR_006 | `INFKGIALMAKKLL` | SEED-006 | 0.812 | 0.643 | 0.357 | REF-FKK-001 (klaklak_analog) | ✓ NOVEL | +| 13 | SEED-008_VAR_032 | `FPVTWRFWRWWKG` | SEED-008 | 0.857 | 0.692 | 0.308 | REF-RRW-001 (tachyplesin_like) | ✓ NOVEL | +| 14 | SEED-008_VAR_009 | `FPITWRWFKWWKG` | SEED-008 | 0.849 | 0.692 | 0.308 | REF-RRW-001 (tachyplesin_like) | ✓ NOVEL | +| 15 | SEED-008_VAR_019 | `FPVSWRWWKFWKG` | SEED-008 | 0.845 | 0.692 | 0.308 | REF-IND-001 (indolicidin) | ✓ NOVEL | +| 16 | SEED-003_VAR_017 | `RRWNWRMKKMG` | SEED-003 | 0.816 | 0.182 | 0.818 | REF-RRW-001 (tachyplesin_like) | ⚠ KNOWN | +| 17 | SEED-003_VAR_012 | `RKWQYRMKKLG` | SEED-003 | 0.807 | 0.182 | 0.818 | REF-RRW-001 (tachyplesin_like) | ⚠ KNOWN | +| 18 | SEED-008_VAR_044 | `FPVTWRWWKWYRG` | SEED-008 | 0.832 | 0.600 | 0.400 | REF-IND-002 (indolicidin_analog) | ✓ NOVEL | +| 19 | SEED-005_VAR_019 | `KRLFKKAGSALKFL` | SEED-005 | 0.808 | 0.400 | 0.600 | REF-KWK-001 (template_seed_1) | ≈ CLOSE | +| 20 | SEED-001_VAR_064 | `KWKLFRKIGAVLRVL` | SEED-001 | 0.802 | 0.133 | 0.867 | REF-KWK-001 (template_seed_1) | ⚠ KNOWN | + +## Interpretation + +### KNOWN_VARIANT candidates + +**SEED-003_VAR_017** (`RRWNWRMKKMG`) — 81.8% similar to REF-RRW-001 (tachyplesin_like; Tam_2002_J_Biol_Chem). +The published AMP provides strong activity precedent for this candidate. +Wet-lab value: confirms assay platform works; novelty claim limited. + +**SEED-003_VAR_012** (`RKWQYRMKKLG`) — 81.8% similar to REF-RRW-001 (tachyplesin_like; Tam_2002_J_Biol_Chem). +The published AMP provides strong activity precedent for this candidate. +Wet-lab value: confirms assay platform works; novelty claim limited. + +**SEED-001_VAR_064** (`KWKLFRKIGAVLRVL`) — 86.7% similar to REF-KWK-001 (template_seed_1; Oren_1997_Biochemistry). +The published AMP provides strong activity precedent for this candidate. +Wet-lab value: confirms assay platform works; novelty claim limited. + +### CLOSE_RELATIVE candidates + +**SEED-005_VAR_019** (`KRLFKKAGSALKFL`) — 60.0% similar to REF-KWK-001 (template_seed_1). +Related to a known AMP but with meaningful sequence differences. Activity probability elevated; novelty moderate. + +### NOVEL candidates + +16 candidates have <50% similarity to any sequence in the 72-sequence reference database. +These represent the most scientifically novel fraction of the panel. +Discovery of activity in this subset would be publishable as a novel AMP class. + +## Recommendations + +1. **KNOWN_VARIANT candidates** should be de-emphasised in novelty claims but retained as positive controls that validate the assay platform. +2. **NOVEL candidates** (from seeds: SEED-006, SEED-007, SEED-008, SEED-009) are the primary discovery targets. Any confirmed activity in this subset is publishable as a novel AMP. +3. This report should be submitted with the ASSAY_PREREGISTRATION.md to document the novelty status of all candidates before wet-lab results are seen. + +## Caveat + +This comparison uses a curated 72-sequence reference database. A full BLASTp search against APD3 (>3,800 sequences), DRAMP v3.0 (>22,000 sequences), or dbAMP would be more comprehensive. Perform APD3 BLASTp of NOVEL candidates before publication to confirm novelty claims. See ROADMAP.md §'Beyond v1.0'. diff --git a/src/openamp_foundry/cli.py b/src/openamp_foundry/cli.py index d18f8bea..e00a3201 100644 --- a/src/openamp_foundry/cli.py +++ b/src/openamp_foundry/cli.py @@ -360,6 +360,43 @@ def build_parser() -> argparse.ArgumentParser: help="Similarity threshold for diversity clustering (default: 0.80)", ) + novelty_broad = sub.add_parser( + "novelty-check-broad", + help=( + "Check candidate novelty against a curated AMP reference database (72+ sequences). " + "The standard novelty score compares against 45 seed sequences only; this command " + "compares against the full amp_curated_references.csv to detect near-copies of " + "published AMPs that the seed-based score may miss." + ), + ) + novelty_broad.add_argument( + "--panel-csv", + required=True, + help="Pilot panel CSV (candidate_id + sequence columns required).", + ) + novelty_broad.add_argument( + "--references-csv", + default="examples/known_reference/amp_curated_references.csv", + help="Curated AMP reference CSV (default: amp_curated_references.csv).", + ) + novelty_broad.add_argument( + "--out", + default="outputs/novelty_broad_report.md", + help="Output markdown report path.", + ) + novelty_broad.add_argument( + "--threshold-known", + type=float, + default=0.70, + help="Similarity >= this value → KNOWN_VARIANT (default: 0.70).", + ) + novelty_broad.add_argument( + "--threshold-close", + type=float, + default=0.50, + help="Similarity >= this value → CLOSE_RELATIVE (default: 0.50).", + ) + return parser @@ -419,6 +456,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "diversity-check": return _run_diversity_check(args) + if args.command == "novelty-check-broad": + return _run_novelty_check_broad(args) + parser.error("unknown command") return 2 @@ -1280,5 +1320,251 @@ def _run_diversity_check(args: argparse.Namespace) -> int: return 0 +def _run_novelty_check_broad(args: argparse.Namespace) -> int: + """Compare pilot panel candidates against a curated AMP reference database. + + The standard novelty score compares against ~45 seed sequences only. This command + uses a broader curated reference set (default: 72 published AMPs) to detect whether + any candidates are near-copies of known AMPs that the seed-based score missed. + + Classification thresholds (Levenshtein similarity): + KNOWN_VARIANT >= threshold_known (default 0.70): effectively a known AMP + CLOSE_RELATIVE >= threshold_close (default 0.50): close relative of known AMP + NOVEL < threshold_close: no close match in reference database + """ + import csv as _csv + from openamp_foundry.scoring.novelty import normalized_similarity + from datetime import datetime, timezone + + # Load panel + panel_path = Path(args.panel_csv) + if not panel_path.exists(): + print(json.dumps({"status": "error", "message": f"panel CSV not found: {args.panel_csv}"})) + return 1 + + with open(panel_path, newline="", encoding="utf-8") as f: + panel = list(_csv.DictReader(f)) + + if not panel: + print(json.dumps({"status": "error", "message": "panel CSV is empty"})) + return 1 + + # Load references + refs_path = Path(args.references_csv) + if not refs_path.exists(): + print(json.dumps({"status": "error", "message": f"references CSV not found: {args.references_csv}"})) + return 1 + + refs: list[dict] = [] + seen_seqs: set[str] = set() + with open(refs_path, newline="", encoding="utf-8") as f: + for row in _csv.DictReader(f): + seq = row.get("sequence", "").strip().upper() + if seq and seq not in seen_seqs: + seen_seqs.add(seq) + refs.append({ + "id": row.get("id", row.get("candidate_id", "?")), + "sequence": seq, + "family": row.get("family", "unknown"), + "reference": row.get("reference", ""), + }) + + if not refs: + print(json.dumps({"status": "error", "message": "references CSV has no valid sequences"})) + return 1 + + thr_known = args.threshold_known + thr_close = args.threshold_close + + if thr_close >= thr_known: + print(json.dumps({ + "status": "error", + "message": ( + f"threshold-close ({thr_close}) must be less than threshold-known ({thr_known}); " + "the classification cascade would make CLOSE_RELATIVE unreachable" + ), + })) + return 1 + + import sys as _sys + + # Score each candidate + results = [] + n_known = 0 + n_close = 0 + n_novel = 0 + + for row in panel: + cid = row.get("candidate_id", "?") + seq = row.get("sequence", "").strip().upper() + if not seq: + print(f"WARNING: candidate {cid!r} has an empty sequence — skipping", file=_sys.stderr) + continue + seed = row.get("seed", "?") + seed_novelty = float(row.get("novelty") or 0.0) + ensemble = float(row.get("ensemble") or 0.0) + + best_sim = -1.0 + best_ref: dict = {} + for ref in refs: + s = normalized_similarity(seq, ref["sequence"]) + if s > best_sim: + best_sim = s + best_ref = ref + + best_sim = max(best_sim, 0.0) + + broad_novelty = round(1.0 - best_sim, 4) + + if best_sim >= thr_known: + category = "KNOWN_VARIANT" + n_known += 1 + elif best_sim >= thr_close: + category = "CLOSE_RELATIVE" + n_close += 1 + else: + category = "NOVEL" + n_novel += 1 + + results.append({ + "candidate_id": cid, + "sequence": seq, + "seed": seed, + "ensemble": ensemble, + "seed_novelty": seed_novelty, + "broad_similarity": round(best_sim, 4), + "broad_novelty": broad_novelty, + "best_match_id": best_ref.get("id", ""), + "best_match_seq": best_ref.get("sequence", ""), + "best_match_family": best_ref.get("family", ""), + "best_match_reference": best_ref.get("reference", ""), + "category": category, + }) + + # Build markdown report + ts = datetime.now(timezone.utc).isoformat() + lines = [ + "# Broad Novelty Check — Pilot Panel vs Curated AMP Database", + "", + f"> Generated: {ts} ", + f"> Panel: {args.panel_csv} ", + f"> Reference database: {args.references_csv} ({len(refs)} unique sequences) ", + f"> Thresholds: KNOWN_VARIANT ≥ {thr_known:.0%}, CLOSE_RELATIVE ≥ {thr_close:.0%}", + "", + "## Purpose", + "", + "The standard pipeline novelty score compares candidates against ~45 seed sequences.", + "This report extends the comparison to the curated AMP reference database to detect", + "near-copies of published AMPs that the seed-based novelty score may miss.", + "", + "## Summary", + "", + "| Category | Count | Description |", + "|---|:---:|---|", + f"| KNOWN_VARIANT | {n_known} | ≥{thr_known:.0%} similar to a known published AMP |", + f"| CLOSE_RELATIVE | {n_close} | {thr_close:.0%}–{thr_known:.0%} similar to a known AMP |", + f"| NOVEL | {n_novel} | <{thr_close:.0%} similar — no close match in reference database |", + f"| **Total** | **{len(results)}** | |", + "", + "## Per-Candidate Results", + "", + "| Rank | Candidate | Sequence | Seed | Ensemble | Seed-Novelty | Broad-Sim | Best-Match | Category |", + "|--:|---|---|---|:---:|:---:|:---:|---|---|", + ] + + for i, r in enumerate(results, 1): + cat_icon = {"KNOWN_VARIANT": "⚠ KNOWN", "CLOSE_RELATIVE": "≈ CLOSE", "NOVEL": "✓ NOVEL"}[r["category"]] + match_str = f"{r['best_match_id']} ({r['best_match_family']})" if r["best_match_id"] else "—" + lines.append( + f"| {i} | {r['candidate_id']} | `{r['sequence']}` | {r['seed']} " + f"| {r['ensemble']:.3f} | {r['seed_novelty']:.3f} | {r['broad_similarity']:.3f} " + f"| {match_str} | {cat_icon} |" + ) + + lines += [ + "", + "## Interpretation", + "", + "### KNOWN_VARIANT candidates", + "", + ] + + known_rows = [r for r in results if r["category"] == "KNOWN_VARIANT"] + if known_rows: + for r in known_rows: + lines += [ + f"**{r['candidate_id']}** (`{r['sequence']}`) — {r['broad_similarity']:.1%} similar " + f"to {r['best_match_id']} ({r['best_match_family']}; {r['best_match_reference']}). ", + f"The published AMP provides strong activity precedent for this candidate.", + f"Wet-lab value: confirms assay platform works; novelty claim limited.", + "", + ] + else: + lines += ["No KNOWN_VARIANT candidates found.", ""] + + lines += [ + "### CLOSE_RELATIVE candidates", + "", + ] + close_rows = [r for r in results if r["category"] == "CLOSE_RELATIVE"] + if close_rows: + for r in close_rows: + lines += [ + f"**{r['candidate_id']}** (`{r['sequence']}`) — {r['broad_similarity']:.1%} similar " + f"to {r['best_match_id']} ({r['best_match_family']}). ", + f"Related to a known AMP but with meaningful sequence differences. " + f"Activity probability elevated; novelty moderate.", + "", + ] + else: + lines += ["No CLOSE_RELATIVE candidates found.", ""] + + lines += [ + "### NOVEL candidates", + "", + f"{n_novel} candidates have <{thr_close:.0%} similarity to any sequence in the " + f"{len(refs)}-sequence reference database. ", + "These represent the most scientifically novel fraction of the panel.", + "Discovery of activity in this subset would be publishable as a novel AMP class.", + "", + "## Recommendations", + "", + "1. **KNOWN_VARIANT candidates** should be de-emphasised in novelty claims but retained " + "as positive controls that validate the assay platform.", + ] + + novel_seeds = sorted({r["seed"] for r in results if r["category"] == "NOVEL"}) + novel_seed_str = ", ".join(novel_seeds) if novel_seeds else "none" + lines += [ + f"2. **NOVEL candidates** (from seeds: {novel_seed_str}) are the primary discovery " + "targets. Any confirmed activity in this subset is publishable as a novel AMP.", + "3. This report should be submitted with the ASSAY_PREREGISTRATION.md to document " + "the novelty status of all candidates before wet-lab results are seen.", + "", + "## Caveat", + "", + "This comparison uses a curated 72-sequence reference database. A full BLASTp search " + "against APD3 (>3,800 sequences), DRAMP v3.0 (>22,000 sequences), or dbAMP would be " + "more comprehensive. Perform APD3 BLASTp of NOVEL candidates before publication to " + "confirm novelty claims. See ROADMAP.md §'Beyond v1.0'.", + ] + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines) + "\n", encoding="utf-8") + + summary = { + "status": "ok", + "n_candidates": len(results), + "n_references": len(refs), + "n_known_variant": n_known, + "n_close_relative": n_close, + "n_novel": n_novel, + "out": args.out, + } + print(json.dumps(summary, indent=2)) + return 0 + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/tests/test_cli.py b/tests/test_cli.py index b05e11d3..b2f90f0f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -390,3 +390,91 @@ def test_diversity_check_creates_report(self, tmp_path): assert "Diversity" in text assert "SEED-009_VAR_033" in text assert "SEED-007_VAR_009" in text + + +class TestNoveltyCheckBroad: + def test_novelty_check_broad_returns_zero(self, tmp_path, capsys): + panel_csv = _write_panel(tmp_path) + rc = main([ + "novelty-check-broad", + "--panel-csv", panel_csv, + "--out", str(tmp_path / "novelty.md"), + ]) + assert rc == 0 + + def test_novelty_check_broad_creates_report(self, tmp_path): + panel_csv = _write_panel(tmp_path) + out = tmp_path / "novelty.md" + main([ + "novelty-check-broad", + "--panel-csv", panel_csv, + "--out", str(out), + ]) + assert out.exists() + text = out.read_text() + assert "Broad Novelty Check" in text + assert "SEED-009_VAR_033" in text + assert "SEED-007_VAR_009" in text + + def test_novelty_check_broad_stdout_summary(self, tmp_path, capsys): + panel_csv = _write_panel(tmp_path) + main([ + "novelty-check-broad", + "--panel-csv", panel_csv, + "--out", str(tmp_path / "novelty.md"), + ]) + captured = capsys.readouterr() + data = json.loads(captured.out) + assert data["status"] == "ok" + assert data["n_candidates"] == 2 + assert data["n_references"] > 0 + # Both SEED-009 and SEED-007 variants are genuinely novel (<50% to any reference) + assert data["n_novel"] == 2 + + def test_novelty_check_broad_missing_panel_returns_error(self, tmp_path, capsys): + rc = main([ + "novelty-check-broad", + "--panel-csv", str(tmp_path / "nonexistent.csv"), + "--out", str(tmp_path / "novelty.md"), + ]) + assert rc == 1 + captured = capsys.readouterr() + data = json.loads(captured.out) + assert data["status"] == "error" + + def test_novelty_check_broad_known_variant_classification(self, tmp_path, capsys): + # Craft a reference that is 100% identical to PANEL_CSV_ROW1's sequence + # so SEED-009_VAR_033 is classified as KNOWN_VARIANT at threshold 0.70 + ref_csv = tmp_path / "refs.csv" + ref_csv.write_text( + "id,sequence,family,reference\n" + "KNOWN-001,RRLPRPGYMPRP,test_family,test_ref\n", # exact match to SEED-009_VAR_033 + encoding="utf-8", + ) + panel_csv = _write_panel(tmp_path, two_rows=False) # single row: SEED-009_VAR_033 + main([ + "novelty-check-broad", + "--panel-csv", panel_csv, + "--references-csv", str(ref_csv), + "--out", str(tmp_path / "novelty.md"), + ]) + captured = capsys.readouterr() + data = json.loads(captured.out) + assert data["status"] == "ok" + assert data["n_known_variant"] == 1 + assert data["n_novel"] == 0 + + def test_novelty_check_broad_inverted_thresholds_returns_error(self, tmp_path, capsys): + panel_csv = _write_panel(tmp_path) + rc = main([ + "novelty-check-broad", + "--panel-csv", panel_csv, + "--out", str(tmp_path / "novelty.md"), + "--threshold-known", "0.30", + "--threshold-close", "0.70", # close > known → invalid + ]) + assert rc == 1 + captured = capsys.readouterr() + data = json.loads(captured.out) + assert data["status"] == "error" + assert "threshold-close" in data["message"]