diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe388d..225c6a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,8 @@ jobs: - name: Install dependencies run: python -m pip install --upgrade pip && python -m pip install -r requirements.txt - name: Compile source - run: python -m compileall -q demo.py rag ingest eval + run: python -m compileall -q demo.py rag ingest eval scripts - name: Run unit tests run: python -m pytest -q + - name: Verify result-derived README graphic + run: python scripts/generate_readme_assets.py --check diff --git a/README.md b/README.md index 3ca8c9f..f163e24 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,13 @@ **A fully local cybersecurity RAG prototype with hybrid retrieval, MITRE ATT&CK graph grounding, cited answers, and a repeatable evaluation harness.** +![Paired CyberRAG pilot plot showing keyword coverage for the local-only and retrieval-augmented answer to each of 15 questions](docs/assets/cyberrag-pilot-coverage.svg) + +*Generated from the checked-in 15-question pilot snapshot. Mean deterministic +keyword coverage increased from 0.627 to 0.843, expected documents appeared in +the top five for 14/15 questions, and mean latency increased by 2.694 seconds. +This is a small local pilot, not production validation or cloud-model parity.* + I built CyberRAG to explore a practical question: how much can retrieval improve a small local model on threat-intelligence tasks while keeping query-time data on the user's machine? @@ -140,6 +147,12 @@ python -m pytest -q The unit tests cover tokenization, stopword handling, exact security-identifier extraction, reciprocal-rank fusion, and deterministic evaluation metrics. Full end-to-end evaluation additionally requires Ollama plus generated corpus/index files. +Verify that the README benchmark graphic still matches the checked-in snapshot: + +```bash +python scripts/generate_readme_assets.py --check +``` + ## Repository boundaries Included: diff --git a/docs/assets/cyberrag-pilot-coverage.svg b/docs/assets/cyberrag-pilot-coverage.svg new file mode 100644 index 0000000..1420692 --- /dev/null +++ b/docs/assets/cyberrag-pilot-coverage.svg @@ -0,0 +1,127 @@ + + +CyberRAG paired pilot keyword coverage +A paired dot plot for 15 cybersecurity questions. Mean keyword coverage rose from 0.627 without retrieval to 0.843 with CyberRAG; context retrieval hit 14 of 15 expected documents and mean latency increased by 2.694 seconds. + + + + + + + + + + + +CYBERRAG / PAIRED LOCAL PILOT +Did retrieval help? +Question-level evidence, including ties and regressions. ++21.6 pp +mean keyword coverage +14/15 +expected-document hits ++2.69s +mean latency trade-off + +8 better +6 tied +1 lower +per-question deterministic keyword score +Coverage by question + +local only + +CyberRAG + +0.0 + +0.2 + +0.4 + +0.6 + +0.8 + +1.0 +ssti-rce + + + + +t1059 + + + + +log4shell + + + + +apt29-tech + + + + +sqli-union + + + + +idor + + + + +ssrf + + + + +kerberoast + + + + +persistence-registry + + + + +xxe + + + + +deserialization + + + + +lateral-smb + + + + +phishing-tactic + + + +× +credential-dumping + + + + +oauth-bypass + + + + + +● expected document retrieved in top 5 × miss +15-question pilot • deterministic substring coverage • model-judge score excluded here because the legacy artifact did not record its backend +Not production validation + + diff --git a/scripts/generate_readme_assets.py b/scripts/generate_readme_assets.py new file mode 100644 index 0000000..b023e3c --- /dev/null +++ b/scripts/generate_readme_assets.py @@ -0,0 +1,221 @@ +"""Render CyberRAG's README benchmark graphic from the checked-in pilot result.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import statistics +import sys +import xml.etree.ElementTree as ET +from html import escape +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +RESULT = ROOT / "eval" / "results_2026-06-22_1901.json" +OUTPUT = ROOT / "docs" / "assets" / "cyberrag-pilot-coverage.svg" + + +def _load_result() -> dict[str, Any]: + data = json.loads(RESULT.read_text(encoding="utf-8")) + summary = data.get("summary") + rows = data.get("rows") + if not isinstance(summary, dict) or not isinstance(rows, list) or not rows: + raise ValueError("pilot result is missing its summary or paired rows") + + required = ("keyword_coverage", "latency_s") + for index, row in enumerate(rows): + if not isinstance(row, dict) or not isinstance(row.get("id"), str): + raise ValueError(f"invalid pilot row {index}") + for arm in ("local_only", "cyberrag"): + record = row.get(arm) + if not isinstance(record, dict): + raise ValueError(f"row {index} is missing {arm}") + for metric in required: + value = record.get(metric) + if not isinstance(value, (int, float)): + raise ValueError(f"row {index} has no numeric {arm}.{metric}") + coverage = float(record["keyword_coverage"]) + if not 0.0 <= coverage <= 1.0: + raise ValueError("keyword coverage must remain within [0, 1]") + hit = row["cyberrag"].get("context_hit") + if hit not in (0, 1): + raise ValueError(f"row {index} has no binary context_hit") + + if summary.get("n") != len(rows): + raise ValueError("pilot summary count does not match its rows") + for arm in ("local_only", "cyberrag"): + recorded = summary.get(arm) + if not isinstance(recorded, dict): + raise ValueError(f"pilot summary is missing {arm}") + for metric in required: + recomputed = round(statistics.fmean(row[arm][metric] for row in rows), 3) + if recorded.get(metric) != recomputed: + raise ValueError( + f"pilot summary {arm}.{metric} does not match its rows" + ) + context_hit_rate = round( + statistics.fmean(row["cyberrag"]["context_hit"] for row in rows), 3 + ) + if summary["cyberrag"].get("context_hit_rate") != context_hit_rate: + raise ValueError("pilot summary context_hit_rate does not match its rows") + return data + + +def _svg(data: dict[str, Any]) -> str: + summary = data["summary"] + rows = data["rows"] + local_coverage = float(summary["local_only"]["keyword_coverage"]) + rag_coverage = float(summary["cyberrag"]["keyword_coverage"]) + local_latency = float(summary["local_only"]["latency_s"]) + rag_latency = float(summary["cyberrag"]["latency_s"]) + coverage_delta_pp = (rag_coverage - local_coverage) * 100 + latency_delta = rag_latency - local_latency + context_hits = sum(int(row["cyberrag"]["context_hit"]) for row in rows) + improved = sum( + row["cyberrag"]["keyword_coverage"] > row["local_only"]["keyword_coverage"] + for row in rows + ) + tied = sum( + row["cyberrag"]["keyword_coverage"] == row["local_only"]["keyword_coverage"] + for row in rows + ) + lower = len(rows) - improved - tied + + width = 1200 + height = 790 + plot_left = 595 + plot_right = 1124 + plot_width = plot_right - plot_left + row_start = 260 + row_step = 30 + + lines = [ + '', + f'', + 'CyberRAG paired pilot keyword coverage', + ( + f'A paired dot plot for {len(rows)} cybersecurity questions. ' + f"Mean keyword coverage rose from {local_coverage:.3f} without retrieval to " + f"{rag_coverage:.3f} with CyberRAG; context retrieval hit {context_hits} of " + f"{len(rows)} expected documents and mean latency increased by {latency_delta:.3f} seconds." + ), + '', + '', + '', + '', + '', + "", + '', + '', + '', + "", + '', + 'CYBERRAG / PAIRED LOCAL PILOT', + 'Did retrieval help?', + 'Question-level evidence, including ties and regressions.', + f'+{coverage_delta_pp:.1f} pp', + 'mean keyword coverage', + f'{context_hits}/{len(rows)}', + 'expected-document hits', + f'+{latency_delta:.2f}s', + 'mean latency trade-off', + '', + f'{improved} better', + f'{tied} tied', + f'{lower} lower', + 'per-question deterministic keyword score', + 'Coverage by question', + '', + 'local only', + '', + 'CyberRAG', + ] + + for tick in range(0, 6): + value = tick / 5 + x = plot_left + value * plot_width + lines.extend( + [ + f'', + f'{value:.1f}', + ] + ) + + for index, row in enumerate(rows): + y = row_start + index * row_step + local = float(row["local_only"]["keyword_coverage"]) + rag = float(row["cyberrag"]["keyword_coverage"]) + local_x = plot_left + local * plot_width + rag_x = plot_left + rag * plot_width + direction = ( + "#35d2cf" if rag > local else "#f47e72" if rag < local else "#6e8d94" + ) + lines.extend( + [ + f'{escape(row["id"])}', + f'', + f'', + f'', + ] + ) + if row["cyberrag"]["context_hit"] == 0: + lines.append( + f'×' + ) + else: + lines.append( + f'' + ) + + footer_y = row_start + len(rows) * row_step + 28 + lines.extend( + [ + f'', + f'● expected document retrieved in top 5 × miss', + '15-question pilot • deterministic substring coverage • model-judge score excluded here because the legacy artifact did not record its backend', + 'Not production validation', + "", + "", + "", + ] + ) + return "\n".join(lines) + + +def _validate_svg(content: str) -> None: + try: + ET.fromstring(content) + except ET.ParseError as exc: + raise ValueError(f"generated invalid SVG: {exc}") from exc + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="fail if the checked-in SVG differs from the pilot-derived output", + ) + args = parser.parse_args() + + content = _svg(_load_result()) + _validate_svg(content) + if args.check: + if not OUTPUT.exists() or OUTPUT.read_text(encoding="utf-8") != content: + print(f"stale: {OUTPUT.relative_to(ROOT)}", file=sys.stderr) + return 1 + print("CyberRAG README benchmark asset is current and valid XML.") + return 0 + + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + OUTPUT.write_text(content, encoding="utf-8", newline="\n") + digest = hashlib.sha256(content.encode("utf-8")).hexdigest() + print(f"wrote {OUTPUT.relative_to(ROOT)} sha256={digest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())