From 052e84a4ed3ccc86598d57a23dd1a2a66d1d5685 Mon Sep 17 00:00:00 2001
From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com>
Date: Fri, 5 Jun 2026 23:31:52 +0200
Subject: [PATCH] feat(eval): retrieval-centric capability-discovery eval
 harness (ir.eval)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add `ir.eval` — a deterministic, offline scoring layer for "does retrieval
surface the right capability?". Retrieval-centric by design: ir's corpora are
documents (skills/packages/reports) joined on artifact_id, not a typed
function-calling registry, so the eval scores retrieval of the right artifact,
not argument-dict matching.

Reuse over reinvention: metric math (recall@k / NDCG@k / MRR / MAP) and the
BEIR driver come from `ef.evaluation`; ir adapts a corpus to ef's retriever
contract (query -> [artifact_id]) and reads back a RetrievalEvalReport.

Surface:
- DiscoveryCase (+ save/load JSONL with optional corpus-version meta header)
- as_doc_retriever / to_qrels / retrieval_report (pure-ef path)
- evaluate_discovery -> DiscoveryReport (metrics + failure taxonomy +
  optional score-threshold abstention proxy; records ranking mode and flags a
  hybrid->dense fallback when vd is absent)
- distractor_robustness_curve (seeded needle-in-a-haystack; x-axis capped to
  the corpus so it never over-claims N) + distractor_curve_from_cases
- validate_cases (drift detection: gold ids that left the corpus)
- CLI: `ir eval <corpus> <cases.jsonl> [--mode] [--k]`

Scoring-harness-first: no LLM, no network (dense path is numpy-only;
lexical/hybrid use the already-declared vd dep). Case generation
(back-translation + name-masking) is deferred to a follow-up.

24 hermetic tests (light embedder, disjoint-vocab corpus for exact assertions;
a confusable corpus to prove the distractor curve actually declines) + a demo
fixture. Full suite green; ruff clean.

Refs #12, #1.
---
 ir/__init__.py                  |   6 +
 ir/cli.py                       |  32 +-
 ir/eval.py                      | 681 ++++++++++++++++++++++++++++++++
 tests/fixtures/demo_cases.jsonl |   6 +
 tests/test_eval.py              | 341 ++++++++++++++++
 5 files changed, 1065 insertions(+), 1 deletion(-)
 create mode 100644 ir/eval.py
 create mode 100644 tests/fixtures/demo_cases.jsonl
 create mode 100644 tests/test_eval.py

diff --git a/ir/__init__.py b/ir/__init__.py
index 61ff3cc..ce222a2 100644
--- a/ir/__init__.py
+++ b/ir/__init__.py
@@ -71,3 +71,9 @@ def search(corpus, query, **kwargs):
     if isinstance(corpus, str):
         corpus = open_corpus(corpus)
     return _search(corpus, query, **kwargs)
+
+
+# The evaluation harness is reachable as ``ir.eval`` (its ``ef`` imports are
+# lazy, so this does not weigh down ``import ir``). Kept out of ``__all__`` so a
+# star-import does not shadow the ``eval`` builtin.
+from . import eval  # noqa: E402,F401  (submodule attribute: ir.eval)
diff --git a/ir/cli.py b/ir/cli.py
index 3bd176f..1ad18eb 100644
--- a/ir/cli.py
+++ b/ir/cli.py
@@ -9,6 +9,7 @@
     ir info packages                # config + stats for a corpus
     ir register notes files --root ~/notes --pattern '.*\\.md$'
     ir rm notes                     # unregister (keeps built data)
+    ir eval skills skills_eval.jsonl --mode hybrid   # score retrieval on a case file
 """
 
 from __future__ import annotations
@@ -83,4 +84,33 @@ def rm(name):
     return f"unregistered {name!r}"
 
 
-COMMANDS = [ls, register, build, search, info, rm]
+def eval(name, cases, *, mode="hybrid", k=10):
+    """Score a built corpus's retrieval against a DiscoveryCase JSONL file.
+
+    cases: path to a JSONL file of cases (see :mod:`ir.eval`); each line is a
+    ``{"query": ..., "gold": [artifact_id, ...]}`` record (empty ``gold`` = an
+    abstention case). Prints recall@k / NDCG@k / MRR / MAP plus the failure-mode
+    taxonomy. mode: dense | lexical | hybrid.
+    """
+    from .eval import evaluate_discovery, load_cases, validate_cases
+
+    corpus = open_corpus(name)
+    if len(corpus) == 0:
+        return f"corpus {name!r} is empty; build it first: ir build {name}"
+    case_list = load_cases(cases)
+    if not case_list:
+        return f"no cases found in {cases!r}"
+    drift = validate_cases(corpus, case_list)
+    report = evaluate_discovery(
+        corpus, case_list, mode=mode, primary_k=k, k_values=tuple(sorted({1, 5, k}))
+    )
+    out = str(report)
+    if drift:
+        out += (
+            f"\n  WARNING: {len(drift)} case(s) reference gold ids absent from "
+            f"corpus {name!r} (stale fixture?); their misses are not real."
+        )
+    return out
+
+
+COMMANDS = [ls, register, build, search, info, rm, eval]
diff --git a/ir/eval.py b/ir/eval.py
new file mode 100644
index 0000000..4e252d9
--- /dev/null
+++ b/ir/eval.py
@@ -0,0 +1,681 @@
+"""Capability-discovery evaluation — measuring how well a corpus is *found*.
+
+``ir.eval`` answers the question that the rest of ``ir`` raises: *does retrieval
+actually surface the right capability?* It is a thin, deterministic scoring layer
+over the substrate already in place — ``ir.retrieve.search`` for ranking and
+``ef.evaluation`` for the metric mathematics — plus the genuinely new pieces a
+*capability-discovery* eval needs that generic retrieval evaluation does not.
+
+Design stance (retrieval-centric, reuse ``ef``)
+-----------------------------------------------
+``ir``'s corpora are **documents** — skills, packages, reports — each a
+name + description (+ body), joined on a stable ``artifact_id``. They are *not*
+a typed function-calling registry with argument signatures. So the eval that
+fits is **retrieval of the right artifact**, not argument-dict scoring:
+
+- **recall@k / NDCG@k / MRR / MAP** — computed by ``ef.evaluation``'s pure,
+  tested primitives (:func:`ef.evaluation.recall_at_k`, … ) and driver
+  (:func:`ef.evaluation.evaluate_retrieval`). ``ir.eval`` does not reimplement
+  them; it adapts an ``ir`` corpus to ``ef``'s retriever contract and reads back
+  a :class:`ef.evaluation.RetrievalEvalReport`.
+- **distractor-robustness curve** — retrieval accuracy as the catalog grows
+  (1 gold + N−1 distractors). The single most diagnostic capability-discovery
+  metric, and one ``ef`` does not provide.
+- **failure-mode taxonomy** — every gold case is classed ``hit_rank_1`` /
+  ``surfaced_low_rank`` / ``retrieval_miss`` so a headline number decomposes
+  into *why* it is what it is.
+- **abstention** — an optional score-threshold proxy for "no artifact applies"
+  cases. True abstention is a *selection* concern (a selector commits or
+  refuses); here it is a retrieval-side diagnostic, clearly labelled as such.
+
+The unit of evaluation is a :class:`DiscoveryCase`: an intent (``query``) and
+the ``gold`` ``artifact_id``\\ s that should answer it (empty ``gold`` = an
+abstention case). Cases are plain data — :func:`save_cases` / :func:`load_cases`
+round-trip them as JSONL so an evaluation set is a committable, reproducible
+fixture (the corpora themselves are machine-specific and live, so freezing the
+cases is what makes a run repeatable; :func:`validate_cases` flags gold ids that
+have since drifted out of the corpus).
+
+Case *generation* (back-translation from a corpus with name-masking) needs an
+LLM and is deliberately **out of scope here** — this module scores a given set
+of cases offline, with no network and no model download. (Dense scoring needs
+only numpy; ``lexical`` / ``hybrid`` ranking additionally need the optional
+``vd`` dependency — a hybrid run that silently fell back to dense because ``vd``
+was absent is flagged in the report.)
+
+Quick start::
+
+    import ir
+    from ir import eval as ev
+
+    corpus = ir.build(ir.CorpusSource.from_skills())     # or ir.open_corpus("skills")
+    cases = ev.load_cases("skills_eval.jsonl")
+    report = ev.evaluate_discovery(corpus, cases, mode="hybrid")
+    print(report)                                        # NDCG@10 + taxonomy
+
+    # Just the BEIR-shaped metrics (pure ef), e.g. to A/B dense vs hybrid:
+    ev.retrieval_report(corpus, cases, mode="dense").primary    # mean NDCG@10
+    ev.retrieval_report(corpus, cases, mode="hybrid").primary
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from collections import Counter
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable
+
+from .retrieve import search as _search
+
+#: Rank cutoffs reported by default.
+DFLT_K_VALUES = (1, 5, 10)
+
+#: Metrics reported by default — the ``ef.evaluation`` retrieval metric names.
+DFLT_METRICS = ("ndcg", "recall", "precision", "mrr", "map")
+
+#: Default ranking mode for the eval adapter (hybrid is ``ir``'s strongest).
+DFLT_MODE = "hybrid"
+
+#: Catalog sizes swept by :func:`distractor_robustness_curve` (RAG-MCP-style).
+DFLT_DISTRACTOR_SIZES = (1, 4, 8, 16, 32, 64, 128)
+
+#: Defaults for :func:`distractor_robustness_curve` — deliberately *not*
+#: ``DFLT_MODE``: the curve builds throwaway in-memory sub-corpora, so it favours
+#: the fast, always-offline dense + hashing path. Pass ``mode="hybrid"`` /
+#: ``embedder="default"`` to measure the production configuration instead.
+DFLT_CURVE_MODE = "dense"
+DFLT_CURVE_EMBEDDER = "light"
+
+
+def _vd_available() -> bool:
+    """Whether ``vd`` (needed for ``lexical`` / ``hybrid`` ranking) is importable."""
+    try:
+        from vd import bm25_lexical_search, reciprocal_rank_fusion  # noqa: F401
+
+        return True
+    except Exception:
+        return False
+
+
+# =========================================================================== #
+# The unit of evaluation
+# =========================================================================== #
+
+
+@dataclass(frozen=True)
+class DiscoveryCase:
+    """One discovery probe: an intent and the artifact(s) that should answer it.
+
+    Attributes:
+        query: the user intent / search query.
+        gold: the ``artifact_id``\\ s that correctly answer ``query`` (one for a
+            single-answer case, several when capabilities overlap). An **empty**
+            tuple marks an *abstention* case — no artifact applies.
+        corpus: optional name of the corpus the case targets (provenance / for
+            multi-corpus case files).
+        source_id: optional ``artifact_id`` the query was generated from
+            (back-translation provenance).
+        metadata: free-form per-case metadata (difficulty, generator, …).
+    """
+
+    query: str
+    gold: tuple[str, ...] = ()
+    corpus: str | None = None
+    source_id: str | None = None
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    @property
+    def gold_is_none(self) -> bool:
+        """True when no artifact should match — an abstention case."""
+        return not self.gold
+
+    def to_dict(self) -> dict:
+        """JSON-serializable form (omitting empty optional fields)."""
+        out: dict[str, Any] = {"query": self.query, "gold": list(self.gold)}
+        if self.corpus is not None:
+            out["corpus"] = self.corpus
+        if self.source_id is not None:
+            out["source_id"] = self.source_id
+        if self.metadata:
+            out["metadata"] = dict(self.metadata)
+        return out
+
+    @classmethod
+    def from_dict(cls, d: Mapping[str, Any]) -> "DiscoveryCase":
+        """Inverse of :meth:`to_dict`; ``gold`` may be a string or a list."""
+        gold = d.get("gold") or ()
+        if isinstance(gold, str):
+            gold = (gold,)
+        return cls(
+            query=d["query"],
+            gold=tuple(gold),
+            corpus=d.get("corpus"),
+            source_id=d.get("source_id"),
+            metadata=dict(d.get("metadata") or {}),
+        )
+
+
+def save_cases(
+    cases: Iterable[DiscoveryCase],
+    path: str | Path,
+    *,
+    meta: Mapping[str, Any] | None = None,
+) -> None:
+    """Write ``cases`` to a JSONL file (one case per line).
+
+    An optional ``meta`` mapping is written as a leading ``{"__meta__": …}``
+    line — the natural home for a corpus-version anchor that pins the cases to
+    the corpus snapshot they were generated against.
+    """
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as out:
+        if meta is not None:
+            out.write(json.dumps({"__meta__": dict(meta)}) + "\n")
+        for case in cases:
+            out.write(json.dumps(case.to_dict()) + "\n")
+
+
+def load_cases(path: str | Path) -> list[DiscoveryCase]:
+    """Read :class:`DiscoveryCase`\\ s from a JSONL file (skips a ``__meta__`` header)."""
+    cases: list[DiscoveryCase] = []
+    for line in Path(path).read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        obj = json.loads(line)
+        if "__meta__" in obj:
+            continue
+        cases.append(DiscoveryCase.from_dict(obj))
+    return cases
+
+
+# =========================================================================== #
+# Bridge: an ir corpus as an ef retriever
+# =========================================================================== #
+
+
+def _as_corpus(corpus: Any) -> Any:
+    """Resolve a corpus *name* to a built :class:`~ir.index.Corpus` (pass-through otherwise)."""
+    if isinstance(corpus, str):
+        from .index import open_corpus
+
+        return open_corpus(corpus)
+    return corpus
+
+
+def as_doc_retriever(
+    corpus: Any,
+    *,
+    mode: str = DFLT_MODE,
+    surfaces: Iterable[str] | None = None,
+    **search_kw: Any,
+) -> Callable[..., list[str]]:
+    """Adapt an ``ir`` corpus to ``ef``'s retriever contract.
+
+    Returns a callable ``retriever(query, *, limit=10) -> [artifact_id, …]`` —
+    bare doc-id strings, best-ranked first, one per artifact. That is exactly
+    what :func:`ef.evaluation.evaluate_retrieval` consumes (it maps a bare
+    string straight to a document id), so the returned callable can be handed to
+    ``ef`` with no further adaptation.
+
+    Args:
+        corpus: an :class:`~ir.index.Corpus` or a registered corpus *name*.
+        mode: ranking mode — ``"dense"`` / ``"lexical"`` / ``"hybrid"``.
+        surfaces: restrict to these surface kinds (e.g. ``{"description"}``).
+        **search_kw: any other :func:`ir.retrieve.search` keyword (``filter``,
+            ``rrf_k``, ``rerank``, ``bm25``, …).
+    """
+    corpus = _as_corpus(corpus)
+
+    def retriever(query: str, *, limit: int = 10) -> list[str]:
+        hits = _search(
+            corpus,
+            query,
+            k=limit,
+            mode=mode,
+            surfaces=surfaces,
+            per_artifact=True,
+            **search_kw,
+        )
+        return [hit.artifact_id for hit in hits]
+
+    return retriever
+
+
+def to_qrels(
+    cases: Sequence[DiscoveryCase], *, grade: float = 1.0
+) -> tuple[dict[str, str], dict[str, dict[str, float]]]:
+    """Build ``ef``'s ``(queries, qrels)`` from the gold-bearing cases.
+
+    Abstention cases (empty ``gold``) are skipped — ``ef``'s retrieval metrics
+    require at least one positively-judged document per query. Query ids are
+    derived from each case's position so they stay stable across the gap left
+    by skipped abstention cases.
+    """
+    queries: dict[str, str] = {}
+    qrels: dict[str, dict[str, float]] = {}
+    for i, case in enumerate(cases):
+        if case.gold_is_none:
+            continue
+        qid = f"q{i:05d}"
+        queries[qid] = case.query
+        qrels[qid] = {aid: grade for aid in case.gold}
+    return queries, qrels
+
+
+def retrieval_report(
+    corpus: Any,
+    cases: Sequence[DiscoveryCase],
+    *,
+    k_values: Sequence[int] = DFLT_K_VALUES,
+    metrics: Sequence[str] = DFLT_METRICS,
+    mode: str = DFLT_MODE,
+    surfaces: Iterable[str] | None = None,
+    limit: int | None = None,
+    **search_kw: Any,
+) -> Any:
+    """Score a corpus's retrieval against the cases — the pure-``ef`` path.
+
+    A thin wrapper that builds the retriever adapter and hands it, with the
+    cases' ``(queries, qrels)``, to :func:`ef.evaluation.evaluate_retrieval`.
+    Returns its :class:`ef.evaluation.RetrievalEvalReport` (``.primary`` is mean
+    NDCG@10). Use this to A/B configurations — dense vs hybrid, with vs without
+    a reranker — on the standard BEIR/MTEB metrics.
+
+    For the richer report (taxonomy + abstention) use :func:`evaluate_discovery`.
+
+    Raises:
+        ValueError: if no case carries gold (``ef`` requires at least one
+            positively-judged query). :func:`evaluate_discovery` instead
+            tolerates an all-abstention set — the two paths differ only on that
+            degenerate input.
+    """
+    from ef.evaluation import evaluate_retrieval
+
+    corpus = _as_corpus(corpus)
+    queries, qrels = to_qrels(cases)
+    retriever = as_doc_retriever(corpus, mode=mode, surfaces=surfaces, **search_kw)
+    return evaluate_retrieval(
+        retriever,
+        qrels,
+        queries,
+        k_values=tuple(k_values),
+        metrics=tuple(metrics),
+        limit=limit,
+    )
+
+
+# =========================================================================== #
+# Corpus introspection — drift detection
+# =========================================================================== #
+
+
+def corpus_artifact_ids(corpus: Any) -> set[str]:
+    """The set of ``artifact_id``\\ s present in a built corpus."""
+    corpus = _as_corpus(corpus)
+    _ids, _matrix, metas = corpus.store.matrix()
+    return {meta["artifact_id"] for meta in metas}
+
+
+def validate_cases(corpus: Any, cases: Sequence[DiscoveryCase]) -> dict[int, list[str]]:
+    """Gold ids a case references that are absent from the corpus.
+
+    Returns ``{case_index: [missing_artifact_id, …]}`` for every case whose gold
+    has drifted out of the corpus (an empty dict means the cases are aligned).
+    The eval corpora are live and machine-specific, so a committed case file can
+    quietly fall out of sync; run this before trusting a report.
+    """
+    present = corpus_artifact_ids(corpus)
+    missing: dict[int, list[str]] = {}
+    for i, case in enumerate(cases):
+        absent = [aid for aid in case.gold if aid not in present]
+        if absent:
+            missing[i] = absent
+    return missing
+
+
+# =========================================================================== #
+# The rich report: metrics + failure taxonomy + abstention
+# =========================================================================== #
+
+
+def _gold_rank(ranking: Sequence[str], gold: Sequence[str]) -> int:
+    """1-based rank of the best-placed gold id in ``ranking`` (0 if none present)."""
+    gold_set = set(gold)
+    for i, aid in enumerate(ranking):
+        if aid in gold_set:
+            return i + 1
+    return 0
+
+
+@dataclass(frozen=True)
+class DiscoveryReport:
+    """The outcome of :func:`evaluate_discovery`.
+
+    Bundles the standard retrieval metrics with the capability-discovery extras:
+
+    Attributes:
+        retrieval: the :class:`ef.evaluation.RetrievalEvalReport` over the
+            gold-bearing cases.
+        failure_classes: counts per failure class (``hit_rank_1`` /
+            ``surfaced_low_rank`` / ``retrieval_miss`` for gold cases;
+            ``abstention_ok`` / ``false_action`` / ``abstention_unscored`` for
+            abstention cases).
+        n_cases / n_gold / n_abstention: case counts.
+        primary_k: the cutoff ``primary`` reports at (default 10).
+        abstention_accuracy: accuracy on the abstention slice when an
+            ``abstain_threshold`` was supplied, else ``None``.
+        mode: the ranking mode the report was produced with — recorded so a
+            number is never mistaken for a different configuration's.
+        vd_degraded: True when ``mode`` was ``lexical`` / ``hybrid`` but ``vd``
+            was unavailable, so ranking silently fell back to dense (the scores
+            are *not* the requested configuration's).
+    """
+
+    retrieval: Any
+    failure_classes: Mapping[str, int]
+    n_cases: int
+    n_gold: int
+    n_abstention: int
+    primary_k: int = 10
+    abstention_accuracy: float | None = None
+    mode: str = DFLT_MODE
+    vd_degraded: bool = False
+
+    @property
+    def primary(self) -> float | None:
+        """The headline number — mean NDCG@``primary_k`` (``None`` if not computed)."""
+        return self.retrieval.primary_at(self.primary_k)
+
+    def __str__(self) -> str:
+        primary = self.primary
+        head = (
+            f"DiscoveryReport — {self.n_cases} cases "
+            f"({self.n_gold} gold, {self.n_abstention} abstention)"
+        )
+        mode_line = f"  mode: {self.mode}" + (
+            "  (DEGRADED: vd unavailable -> dense fallback)" if self.vd_degraded else ""
+        )
+        lines = [
+            head,
+            mode_line,
+            "  primary NDCG@%d: %s"
+            % (self.primary_k, f"{primary:.4f}" if primary is not None else "n/a"),
+        ]
+        for key, value in self.retrieval.metrics.items():
+            lines.append(f"    {key}: {value:.4f}")
+        if self.failure_classes:
+            lines.append("  failure classes:")
+            for cls, count in sorted(
+                self.failure_classes.items(), key=lambda kv: -kv[1]
+            ):
+                lines.append(f"    {cls}: {count}")
+        if self.abstention_accuracy is not None:
+            lines.append(f"  abstention accuracy: {self.abstention_accuracy:.4f}")
+        return "\n".join(lines)
+
+
+def evaluate_discovery(
+    corpus: Any,
+    cases: Sequence[DiscoveryCase],
+    *,
+    k_values: Sequence[int] = DFLT_K_VALUES,
+    primary_k: int = 10,
+    metrics: Sequence[str] = DFLT_METRICS,
+    mode: str = DFLT_MODE,
+    surfaces: Iterable[str] | None = None,
+    abstain_threshold: float | None = None,
+    **search_kw: Any,
+) -> DiscoveryReport:
+    """Score a corpus against ``cases`` — metrics, failure taxonomy, abstention.
+
+    Runs retrieval **once per case** and, from each ranking, computes the
+    ``ef.evaluation`` metric primitives (so the numbers match the pure-``ef``
+    path) *and* the capability-discovery extras in one pass.
+
+    Args:
+        corpus: an :class:`~ir.index.Corpus` or a registered corpus *name*.
+        k_values: rank cutoffs to report (``primary_k`` is always included).
+        primary_k: the cutoff the headline ``primary`` reports at.
+        metrics: which ``ef`` metrics to compute (``ndcg`` / ``recall`` /
+            ``precision`` / ``mrr`` / ``map``).
+        mode: ranking mode passed to :func:`ir.retrieve.search`.
+        surfaces: restrict to these surface kinds.
+        abstain_threshold: if given, an abstention case is scored *correct* when
+            the top hit's score is below this threshold (a retrieval-side proxy
+            for "no artifact applies"); leave ``None`` to only count abstention
+            cases without scoring them.
+        **search_kw: any other :func:`ir.retrieve.search` keyword.
+
+    Returns:
+        a :class:`DiscoveryReport`. If no case carries gold (an empty or
+        all-abstention set) the retrieval metrics are omitted — ``report.primary``
+        is ``None`` — but the taxonomy and abstention counts are still returned
+        (unlike :func:`retrieval_report`, which raises ``ValueError`` on that
+        input).
+    """
+    from ef.evaluation import (
+        RetrievalEvalReport,
+        average_precision,
+        ndcg_at_k,
+        precision_at_k,
+        recall_at_k,
+        reciprocal_rank,
+    )
+
+    # Deliberate mirror of ef.evaluation._RETRIEVAL_METRICS (private, so it cannot
+    # be imported). The metric *functions* are ef's — only this name->fn map and
+    # the unknown-name check are duplicated; ef exposing a public registry would
+    # let this go away.
+    metric_fns: dict[str, Callable[..., float]] = {
+        "ndcg": ndcg_at_k,
+        "recall": recall_at_k,
+        "precision": precision_at_k,
+        "mrr": reciprocal_rank,
+        "map": average_precision,
+    }
+    unknown = [name for name in metrics if name not in metric_fns]
+    if unknown:
+        raise ValueError(
+            f"Unknown metric(s) {unknown}. Choose from {sorted(metric_fns)}."
+        )
+
+    corpus = _as_corpus(corpus)
+    # primary_k is always reported; sort+dedupe so report keys are canonical.
+    k_values = tuple(sorted(set(k_values) | {primary_k}))
+    limit = max(k_values)
+    vd_degraded = mode in ("hybrid", "lexical") and not _vd_available()
+
+    per_query: dict[str, dict[str, float]] = {}
+    taxonomy: Counter[str] = Counter()
+    n_abstention = 0
+    abstain_correct = 0
+
+    for i, case in enumerate(cases):
+        # per_artifact=True yields one id per artifact (its best surface) — the
+        # same shape ef.evaluate_retrieval produces after its internal _dedup, so
+        # the metric primitives below score the same ranking as the pure-ef path.
+        hits = _search(
+            corpus,
+            case.query,
+            k=limit,
+            mode=mode,
+            surfaces=surfaces,
+            per_artifact=True,
+            **search_kw,
+        )
+        ranking = [hit.artifact_id for hit in hits]
+        top_score = hits[0].score if hits else None
+
+        if case.gold_is_none:
+            n_abstention += 1
+            if abstain_threshold is None:
+                taxonomy["abstention_unscored"] += 1
+            elif top_score is None or top_score < abstain_threshold:
+                taxonomy["abstention_ok"] += 1
+                abstain_correct += 1
+            else:
+                taxonomy["false_action"] += 1
+            continue
+
+        relevant = {aid: 1.0 for aid in case.gold}
+        per_query[f"q{i:05d}"] = {
+            f"{name}@{k}": metric_fns[name](ranking, relevant, k)
+            for name in metrics
+            for k in k_values
+        }
+
+        rank = _gold_rank(ranking, case.gold)
+        if rank == 0 or rank > primary_k:
+            taxonomy["retrieval_miss"] += 1
+        elif rank == 1:
+            taxonomy["hit_rank_1"] += 1
+        else:
+            taxonomy["surfaced_low_rank"] += 1
+
+    metric_keys = [f"{name}@{k}" for name in metrics for k in k_values]
+    n_gold = len(per_query)
+    # With no gold-bearing case, leave metrics empty so RetrievalEvalReport.
+    # primary_at(...) returns None (the "not computed" contract) rather than NaN.
+    aggregated = (
+        {
+            key: sum(scores[key] for scores in per_query.values()) / n_gold
+            for key in metric_keys
+        }
+        if n_gold
+        else {}
+    )
+    retrieval = RetrievalEvalReport(
+        metrics=aggregated,
+        per_query=per_query,
+        n_queries=n_gold,
+        k_values=tuple(k_values),
+    )
+    abstention_accuracy = (
+        abstain_correct / n_abstention
+        if abstain_threshold is not None and n_abstention
+        else None
+    )
+    return DiscoveryReport(
+        retrieval=retrieval,
+        failure_classes=dict(taxonomy),
+        n_cases=len(cases),
+        n_gold=n_gold,
+        n_abstention=n_abstention,
+        primary_k=primary_k,
+        abstention_accuracy=abstention_accuracy,
+        mode=mode,
+        vd_degraded=vd_degraded,
+    )
+
+
+# =========================================================================== #
+# Distractor-robustness curve
+# =========================================================================== #
+
+
+def distractor_robustness_curve(
+    scope: Mapping[str, Any],
+    probes: Sequence[tuple[str, str]],
+    *,
+    strategy: Any = None,
+    embedder: Any = DFLT_CURVE_EMBEDDER,
+    mode: str = DFLT_CURVE_MODE,
+    sizes: Sequence[int] = DFLT_DISTRACTOR_SIZES,
+    trials: int = 20,
+    seed: int = 0,
+    k: int = 1,
+) -> dict[int, float]:
+    """Retrieval accuracy as the catalog grows — the needle-in-a-haystack curve.
+
+    For each catalog size ``N`` and each ``(query, gold_id)`` probe, build many
+    ``N``-artifact sub-corpora (the gold artifact plus ``N−1`` distractors
+    sampled from ``scope``) and measure how often the gold artifact lands in the
+    top ``k``. The accuracy at each ``N`` is averaged over the probes and over
+    ``trials`` random distractor draws. ``N=1`` is the trivial anchor (gold
+    alone) and is always 1.0; a steep decline as ``N`` grows is the signature of
+    retrieval that does not discriminate.
+
+    Sub-corpora are built in-memory with the ``"light"`` (numpy-only) embedder
+    by default, so the curve is fast and offline; pass ``embedder="default"``
+    (and ``mode="hybrid"``) to measure the production configuration.
+
+    Sizes larger than the corpus are **capped** at ``len(scope)`` (the largest
+    catalog the pool can fill) and de-duplicated, so the returned keys are the
+    catalog sizes actually built — the x-axis never claims an ``N`` larger than
+    the corpus could supply.
+
+    Args:
+        scope: the full ``{artifact_id: raw}`` pool to draw the catalog from.
+        probes: ``(query, gold_id)`` pairs; ``gold_id`` must be a key of ``scope``.
+        strategy: the :class:`~ir.strategy.IndexingStrategy` to index with
+            (default :class:`~ir.strategy.WholeText`).
+        embedder: embedder spec for the sub-corpora.
+        mode: ranking mode for the probe searches.
+        sizes: the catalog sizes to sweep (capped at ``len(scope)``, deduped).
+        trials: random distractor draws averaged per size (a size that consumes
+            the whole pool has no sampling freedom, so it runs a single trial).
+        seed: base RNG seed (reproducible; varied per size).
+        k: the top-``k`` the gold must reach to count as a hit.
+
+    Returns:
+        ``{N: accuracy}`` for each *achievable* ``N`` (the requested sizes capped
+        at ``len(scope)`` and de-duplicated).
+    """
+    from .index import build
+    from .sources import CorpusSource
+    from .store import CorpusStore
+    from .strategy import WholeText
+
+    scope = dict(scope)
+    strategy = strategy or WholeText()
+    all_ids = list(scope)
+    max_catalog = len(all_ids)
+    effective_sizes = sorted({min(n, max_catalog) for n in sizes if n >= 1})
+    curve: dict[int, float] = {}
+
+    for size in effective_sizes:
+        rng = random.Random(seed + size)
+        hits = 0
+        total = 0
+        for query, gold_id in probes:
+            if gold_id not in scope:
+                continue
+            pool = [aid for aid in all_ids if aid != gold_id]
+            n_distractors = min(size - 1, len(pool))
+            # The full pool admits no sampling freedom — every trial would build
+            # the same catalog — so one representative trial suffices.
+            n_trials = 1 if n_distractors >= len(pool) else trials
+            for _ in range(n_trials):
+                sample = rng.sample(pool, n_distractors) if n_distractors else []
+                sub = {aid: scope[aid] for aid in (gold_id, *sample)}
+                source = CorpusSource.from_mapping(
+                    sub, name="_distractor", strategy=strategy
+                )
+                corpus = build(source, store=CorpusStore.memory(), embedder=embedder)
+                ranking = [
+                    hit.artifact_id for hit in _search(corpus, query, k=k, mode=mode)
+                ]
+                total += 1
+                if gold_id in ranking[:k]:
+                    hits += 1
+        curve[size] = hits / total if total else float("nan")
+    return curve
+
+
+def distractor_curve_from_cases(
+    source: Any, cases: Sequence[DiscoveryCase], **kwargs: Any
+) -> dict[int, float]:
+    """Convenience: a distractor curve over a source's scope from single-gold cases.
+
+    Probes are the single-gold cases (``(query, gold[0])``); the source's
+    ``scope`` is the distractor pool and its ``indexing_strategy`` is reused
+    unless a ``strategy`` is passed in ``kwargs``.
+    """
+    probes = [(case.query, case.gold[0]) for case in cases if len(case.gold) == 1]
+    kwargs.setdefault("strategy", source.indexing_strategy)
+    return distractor_robustness_curve(source.scope, probes, **kwargs)
diff --git a/tests/fixtures/demo_cases.jsonl b/tests/fixtures/demo_cases.jsonl
new file mode 100644
index 0000000..f3a83e2
--- /dev/null
+++ b/tests/fixtures/demo_cases.jsonl
@@ -0,0 +1,6 @@
+{"__meta__": {"corpus": "demo", "note": "disjoint-vocab demo corpus; gold ids are the demo doc keys"}}
+{"query": "how do I write a script in python", "gold": ["python"], "source_id": "python"}
+{"query": "frontend code that runs in the browser", "gold": ["javascript"], "source_id": "javascript"}
+{"query": "store data in relational tables and run queries", "gold": ["database"], "source_id": "database"}
+{"query": "package an app into a container image", "gold": ["docker"], "source_id": "docker"}
+{"query": "knit a wool sweater with bamboo needles", "gold": []}
diff --git a/tests/test_eval.py b/tests/test_eval.py
new file mode 100644
index 0000000..dc4a167
--- /dev/null
+++ b/tests/test_eval.py
@@ -0,0 +1,341 @@
+"""Eval-harness tests — hermetic: the light (numpy-only) embedder, no network.
+
+The demo corpus uses **disjoint vocabularies** so dense retrieval is
+unambiguous: each query shares content tokens with exactly one document, which
+therefore ranks first. That lets the metric assertions be exact (recall@1 == 1)
+without depending on a downloaded model.
+"""
+
+from pathlib import Path
+
+import pytest
+
+import ir
+from ir import eval as ev
+from ir.store import CorpusStore
+
+# Disjoint-vocab demo corpus; keys match tests/fixtures/demo_cases.jsonl gold ids.
+DOCS = {
+    "python": "python programming language scripting interpreter",
+    "javascript": "javascript browser web frontend dom rendering",
+    "database": "sql relational tables rows columns indexing",
+    "docker": "docker container image kubernetes orchestration",
+}
+FIXTURE = Path(__file__).parent / "fixtures" / "demo_cases.jsonl"
+
+
+def _corpus():
+    src = ir.CorpusSource.from_mapping(DOCS, name="demo", strategy=ir.WholeText())
+    return ir.build(src, store=CorpusStore.memory(), embedder="light")
+
+
+def _gold_cases():
+    return [c for c in ev.load_cases(FIXTURE) if not c.gold_is_none]
+
+
+# --------------------------------------------------------------------------- #
+# DiscoveryCase + (de)serialization
+# --------------------------------------------------------------------------- #
+
+
+def test_case_roundtrip(tmp_path):
+    import json
+
+    cases = [
+        ev.DiscoveryCase("q1", gold=("a",), source_id="a", metadata={"d": 1}),
+        ev.DiscoveryCase("q2", gold=("b", "c")),
+        ev.DiscoveryCase("q3", gold=()),  # abstention
+    ]
+    path = tmp_path / "cases.jsonl"
+    ev.save_cases(cases, path, meta={"corpus": "x"})
+    loaded = ev.load_cases(path)
+    assert loaded == cases  # frozen dataclasses compare by value
+    assert loaded[2].gold_is_none
+    # the meta header is actually written as line 1 and is recoverable
+    first_line = path.read_text(encoding="utf-8").splitlines()[0]
+    assert json.loads(first_line) == {"__meta__": {"corpus": "x"}}
+
+    # omitting meta writes no header line
+    path2 = tmp_path / "no_meta.jsonl"
+    ev.save_cases(cases, path2)
+    assert "__meta__" not in path2.read_text(encoding="utf-8")
+    assert ev.load_cases(path2) == cases
+
+
+def test_from_dict_accepts_scalar_gold():
+    case = ev.DiscoveryCase.from_dict({"query": "q", "gold": "a"})
+    assert case.gold == ("a",)
+
+
+def test_load_fixture_skips_meta_header():
+    cases = ev.load_cases(FIXTURE)
+    assert len(cases) == 5
+    assert sum(c.gold_is_none for c in cases) == 1
+
+
+# --------------------------------------------------------------------------- #
+# Retriever adapter + qrels
+# --------------------------------------------------------------------------- #
+
+
+def test_as_doc_retriever_returns_artifact_ids():
+    retriever = ev.as_doc_retriever(_corpus(), mode="dense")
+    ranking = retriever("python programming", limit=4)
+    assert ranking[0] == "python"
+    assert set(ranking) <= set(DOCS)
+
+
+def test_to_qrels_skips_abstention():
+    cases = ev.load_cases(FIXTURE)
+    queries, qrels = ev.to_qrels(cases)
+    assert len(queries) == 4 and len(qrels) == 4  # the abstention case is dropped
+    assert set(queries) == set(qrels)
+    # ids are position-stable: the abstention case at index 4 leaves a q00004 gap.
+    assert set(qrels) == {"q00000", "q00001", "q00002", "q00003"}
+    assert "q00004" not in qrels and "q00004" not in queries
+
+
+# --------------------------------------------------------------------------- #
+# Metrics (delegating to ef) — exact on the disjoint corpus
+# --------------------------------------------------------------------------- #
+
+
+def test_retrieval_report_perfect_on_disjoint_corpus():
+    report = ev.retrieval_report(_corpus(), _gold_cases(), mode="dense")
+    assert report.n_queries == 4
+    assert report.metrics["recall@1"] == pytest.approx(1.0)
+    assert report.metrics["ndcg@1"] == pytest.approx(1.0)
+    assert report.primary_at(1) == pytest.approx(1.0)
+
+
+def test_retrieval_report_detects_a_miss():
+    # Deliberately mislabel: the query is about docker, gold says database.
+    cases = [ev.DiscoveryCase("container image kubernetes", gold=("database",))]
+    report = ev.retrieval_report(_corpus(), cases, mode="dense", k_values=(1,))
+    assert report.metrics["recall@1"] == pytest.approx(0.0)
+
+
+# --------------------------------------------------------------------------- #
+# evaluate_discovery — report + taxonomy + abstention
+# --------------------------------------------------------------------------- #
+
+
+def test_evaluate_discovery_taxonomy_all_rank1():
+    report = ev.evaluate_discovery(_corpus(), _gold_cases(), mode="dense", primary_k=1)
+    assert report.n_gold == 4 and report.n_abstention == 0
+    assert report.failure_classes.get("hit_rank_1") == 4
+    assert report.primary == pytest.approx(1.0)
+    assert "DiscoveryReport" in str(report)
+
+
+def test_evaluate_discovery_matches_pure_ef_path():
+    cases = _gold_cases()
+    rich = ev.evaluate_discovery(cases=cases, corpus=_corpus(), mode="dense")
+    pure = ev.retrieval_report(_corpus(), cases, mode="dense")
+    for key in pure.metrics:
+        assert rich.retrieval.metrics[key] == pytest.approx(pure.metrics[key])
+
+
+def test_abstention_separation_and_accuracy():
+    corpus = _corpus()
+    cases = ev.load_cases(FIXTURE)
+    # The abstention query shares no tokens with any doc -> near-zero top score.
+    def top_score(query):
+        hits = ir.search(corpus, query, k=1, mode="dense")
+        return hits[0].score if hits else 0.0
+
+    gold_tops = [top_score(c.query) for c in cases if not c.gold_is_none]
+    abstain_top = next(top_score(c.query) for c in cases if c.gold_is_none)
+    assert abstain_top < min(gold_tops)  # the separating signal exists
+
+    threshold = (abstain_top + min(gold_tops)) / 2
+    report = ev.evaluate_discovery(
+        corpus, cases, mode="dense", abstain_threshold=threshold
+    )
+    assert report.n_abstention == 1
+    assert report.abstention_accuracy == pytest.approx(1.0)
+    assert report.failure_classes.get("abstention_ok") == 1
+
+
+def test_abstention_unscored_without_threshold():
+    report = ev.evaluate_discovery(_corpus(), ev.load_cases(FIXTURE), mode="dense")
+    assert report.failure_classes.get("abstention_unscored") == 1
+    assert report.abstention_accuracy is None
+
+
+def test_unknown_metric_raises():
+    with pytest.raises(ValueError):
+        ev.evaluate_discovery(_corpus(), _gold_cases(), metrics=("bogus",))
+
+
+# --------------------------------------------------------------------------- #
+# Drift detection
+# --------------------------------------------------------------------------- #
+
+
+def test_validate_cases_flags_drift():
+    cases = [
+        ev.DiscoveryCase("q", gold=("python",)),      # present
+        ev.DiscoveryCase("q", gold=("gone", "docker")),  # 'gone' absent
+    ]
+    missing = ev.validate_cases(_corpus(), cases)
+    assert missing == {1: ["gone"]}
+
+
+# --------------------------------------------------------------------------- #
+# Distractor-robustness curve
+# --------------------------------------------------------------------------- #
+
+
+def test_distractor_curve_shape_and_anchor():
+    probes = [("python programming language", "python")]
+    curve = ev.distractor_robustness_curve(
+        DOCS, probes, sizes=(1, 2, 4), trials=3, embedder="light", mode="dense"
+    )
+    assert set(curve) == {1, 2, 4}
+    assert curve[1] == pytest.approx(1.0)  # gold alone is always found
+    assert all(0.0 <= v <= 1.0 for v in curve.values())
+    # Disjoint vocab => gold wins regardless of distractors.
+    assert curve[4] == pytest.approx(1.0)
+
+
+def test_distractor_curve_from_cases():
+    src = ir.CorpusSource.from_mapping(DOCS, name="demo", strategy=ir.WholeText())
+    curve = ev.distractor_curve_from_cases(
+        src, _gold_cases(), sizes=(1, 3), trials=2, mode="dense"
+    )
+    assert set(curve) == {1, 3}
+    assert curve[1] == pytest.approx(1.0)
+
+
+def test_distractor_curve_declines_with_confusable_distractors():
+    # Each distractor shares MORE of the query's tokens than gold does, so any
+    # distractor present outranks gold -> accuracy collapses as the catalog grows.
+    scope = {"gold": "alpha gamma"}
+    for i in range(4):
+        scope[f"d{i}"] = "alpha beta delta"
+    curve = ev.distractor_robustness_curve(
+        scope,
+        [("alpha beta", "gold")],
+        sizes=(1, 2, 4),
+        trials=4,
+        embedder="light",
+        mode="dense",
+    )
+    assert curve[1] == pytest.approx(1.0)  # gold alone
+    assert curve[max(curve)] < curve[1]  # a genuine decline (the headline signal)
+
+
+def test_distractor_curve_caps_sizes_to_corpus():
+    # Sizes beyond the corpus must be capped+deduped, not relabelled.
+    curve = ev.distractor_robustness_curve(
+        DOCS, [("python programming", "python")], sizes=(1, 4, 8, 128), trials=2
+    )
+    assert set(curve) == {1, 4}  # corpus has 4 docs; 8 and 128 collapse to 4
+
+
+# --------------------------------------------------------------------------- #
+# Degenerate inputs: no gold-bearing cases
+# --------------------------------------------------------------------------- #
+
+
+def test_no_gold_yields_none_primary_not_nan():
+    cases = [ev.DiscoveryCase("q1", gold=()), ev.DiscoveryCase("q2", gold=())]
+    report = ev.evaluate_discovery(_corpus(), cases, mode="dense")
+    assert report.n_gold == 0
+    assert report.primary is None  # the None-means-not-computed contract holds
+    assert report.retrieval.metrics == {}
+    assert "n/a" in str(report) and "nan" not in str(report).lower()
+
+
+def test_retrieval_report_raises_on_all_abstention():
+    # The pure-ef path raises (ef requires a positively-judged query); the rich
+    # path tolerates it (see test above). The asymmetry is documented.
+    with pytest.raises(ValueError):
+        ev.retrieval_report(_corpus(), [ev.DiscoveryCase("q", gold=())], mode="dense")
+
+
+def test_abstention_false_action_when_threshold_too_low():
+    # A threshold below the abstention query's top score => the system "acts" when
+    # it should abstain: false_action, accuracy 0.
+    report = ev.evaluate_discovery(
+        _corpus(), ev.load_cases(FIXTURE), mode="dense", abstain_threshold=-1.0
+    )
+    assert report.failure_classes.get("false_action") == 1
+    assert report.abstention_accuracy == pytest.approx(0.0)
+
+
+def test_per_query_keys_skip_abstention_index():
+    report = ev.evaluate_discovery(_corpus(), ev.load_cases(FIXTURE), mode="dense")
+    assert set(report.retrieval.per_query) == {"q00000", "q00001", "q00002", "q00003"}
+
+
+# --------------------------------------------------------------------------- #
+# surfaces= restriction threads through the eval path
+# --------------------------------------------------------------------------- #
+
+
+def test_surfaces_restriction_threads_through():
+    docs = {
+        "alpha": {"name": "alpha", "description": "alpha tool for foo tasks"},
+        "beta": {"name": "beta", "description": "beta tool for bar tasks"},
+    }
+    src = ir.CorpusSource.from_mapping(docs, name="skl", strategy=ir.Skill())
+    corpus = ir.build(src, store=CorpusStore.memory(), embedder="light")
+    # Skill surfaces are kind "capability".
+    keep = ev.as_doc_retriever(corpus, mode="dense", surfaces={"capability"})
+    assert keep("alpha tool for foo tasks", limit=2)[0] == "alpha"
+    drop = ev.as_doc_retriever(corpus, mode="dense", surfaces={"nonexistent"})
+    assert drop("alpha tool for foo tasks", limit=2) == []
+
+
+# --------------------------------------------------------------------------- #
+# CLI eval — drift warning, empty corpus, no cases (hermetic, light embedder)
+# --------------------------------------------------------------------------- #
+
+
+def test_cli_eval_paths(tmp_path, monkeypatch):
+    import json
+
+    monkeypatch.setenv("IR_CONFIG_DIR", str(tmp_path / "config"))
+    monkeypatch.setenv("IR_DATA_DIR", str(tmp_path / "data"))
+    monkeypatch.setenv("IR_CACHE_DIR", str(tmp_path / "cache"))
+    from ir import cli
+
+    docs = tmp_path / "docs"
+    docs.mkdir()
+    (docs / "deploy.md").write_text("deploy the app to the server with systemd units")
+    (docs / "baking.md").write_text("bake a cake in the oven with flour and sugar")
+    cli.register("notes", "files", root=str(docs), pattern=r".*\.md$")
+    cli.build("notes", embedder="light")  # built with light -> open_corpus stays offline
+
+    cases = tmp_path / "cases.jsonl"
+    cases.write_text(
+        json.dumps({"query": "deploy app systemd", "gold": ["deploy.md"]})
+        + "\n"
+        + json.dumps({"query": "irrelevant", "gold": ["gone.md"]})  # drift
+        + "\n"
+    )
+    out = cli.eval("notes", str(cases), mode="dense", k=3)
+    assert "DiscoveryReport" in out
+    assert "WARNING" in out  # drift on the absent gold id is surfaced
+
+    empty = tmp_path / "empty.jsonl"
+    empty.write_text("")
+    assert "no cases found" in cli.eval("notes", str(empty), mode="dense")
+
+    blank_dir = tmp_path / "blankdir"
+    blank_dir.mkdir()
+    cli.register("blank", "files", root=str(blank_dir), pattern=r".*\.md$")
+    cli.build("blank", embedder="light")  # 0 records, config pins light embedder
+    assert "build it first" in cli.eval("blank", str(cases), mode="dense")
+
+
+# --------------------------------------------------------------------------- #
+# ir.eval is reachable as a submodule attribute
+# --------------------------------------------------------------------------- #
+
+
+def test_eval_reachable_via_ir_namespace():
+    assert hasattr(ir, "eval")
+    assert ir.eval.DiscoveryCase is ev.DiscoveryCase