Retrieval eval + benchmark harness (tooling)

evals/__init__.py

@@ -0,0 +1 @@
+"""Version-controlled evaluation harnesses; never shipped with the package."""

evals/benchmark/README.md

@@ -0,0 +1,121 @@
+# Benchmark harness: lilbee vs RAGFlow
+
+A preregistered, reproducible A/B of retrieval quality. Two arms answer the
+same public, human-labeled datasets with the same served model, so retrieval is
+the only variable:
+
+- **Arm A - lilbee**: the `test/retrieval-parity` branch with every parity
+  feature enabled.
+- **Arm B - RAGFlow**: the DeepDoc pipeline at its defaults.
+
+One embedder and one generator are served once and pointed at by both arms, so
+generation is identical and only the retrieval stack changes. Lives outside
+`src/` on purpose: it never ships in the package.
+
+## Two tiers
+
+1. **Tier 1 - retrieval, no model opinion.** Each arm returns a ranked list per
+   query, written as a TREC run file. `pytrec_eval` scores it against the
+   dataset's published relevance labels (qrels): nDCG@10, Recall@20, MRR@10.
+   Nothing is graded by a model, so the numbers are bit-for-bit reproducible by
+   anyone with the same run files and qrels.
+2. **Tier 2 - answer quality.** Both arms answer with the same model; RAGAS
+   scores faithfulness, answer relevancy, and context precision/recall. The
+   blind duplicate-arm judge from `evals/retrieval` is reused as a corroborating
+   signal, with its noise floor measured by grading one arm twice.
+
+Every cross-arm difference is paired per query and gets a bootstrap 95% CI and a
+randomization-test p-value. A difference whose CI crosses zero is reported as
+not significant, not as a win.
+
+## Native vs derived labels
+
+Passage sets (BEIR SciFact/FiQA/NFCorpus, HotpotQA, TREC-COVID) ship native TREC
+qrels, used as published. The document-structured QA sets (TAT-DQA, OTT-QA) have
+no retrieval labels of their own, so their qrels are **derived** from human
+gold-evidence annotations by one documented pure function
+(`datasets.derive_qrels_from_evidence`). Every derived dataset is marked
+`label_kind: derived` in the manifest and flagged in the report.
+
+## Pod workflow
+
+One A100-80GB in a named tmux so a reclaimed pod reattaches. Preregister before
+any data moves, then run each arm, score, and power off.
+
+```bash
+cd ~/lilbee  # the repo checkout; run everything from the repo root
+RUN=/tmp/bench
+
+# 0. Freeze the preregistration. Nothing can be cherry-picked after this.
+uv run python -m evals.benchmark preregister \
+  --manifest evals/benchmark/manifest.example.yaml --out "$RUN/manifest.frozen.json"
+
+# 1. Serve the shared model with lilbee (all parity features on) and stand up
+#    RAGFlow pointed at the same OpenAI-compatible endpoint.
+export LILBEE_TITLE_SEARCH=true LILBEE_NEIGHBOR_EXPANSION=2 \
+  LILBEE_TABLE_EXTRACTION=true LILBEE_LAYOUT_DETECTION=true LILBEE_INTENT_LLM=true
+lilbee serve --port 8080 &
+uv run python -m evals.benchmark.bootstrap_ragflow \
+  --base-url http://127.0.0.1:9380 --api-key "$RAGFLOW_KEY" \
+  --corpus-dir "$RUN/corpus" --llm-model qwen2.5-72b-instruct \
+  --embedding-model qwen3-embedding
+
+# 2. Ingest the same corpus into both systems (lilbee add; bootstrap uploaded
+#    RAGFlow's copy above).
+
+# 3. Collect a TREC run file per arm. Each query is checkpointed, so a killed
+#    run resumes.
+uv run python -m evals.benchmark collect --system lilbee \
+  --queries "$RUN/queries.jsonl" --base-url http://127.0.0.1:8080 \
+  --run-tag lilbee --run "$RUN/run-lilbee.trec" --checkpoint "$RUN/ck-lilbee.jsonl"
+uv run python -m evals.benchmark collect --system ragflow \
+  --queries "$RUN/queries.jsonl" --base-url http://127.0.0.1:9380 \
+  --api-key "$RAGFLOW_KEY" --dataset-id "$RAGFLOW_DATASET" \
+  --run-tag ragflow --run "$RUN/run-ragflow.trec" --checkpoint "$RUN/ck-ragflow.jsonl"
+
+# 4. Tier 1: score each run against the qrels with pytrec_eval.
+uv run python -m evals.benchmark score-ir --qrels "$RUN/qrels.json" \
+  --run "$RUN/run-lilbee.trec" --dataset scifact --run-tag lilbee \
+  --out "$RUN/ir-lilbee.jsonl"
+uv run python -m evals.benchmark score-ir --qrels "$RUN/qrels.json" \
+  --run "$RUN/run-ragflow.trec" --dataset scifact --run-tag ragflow \
+  --out "$RUN/ir-ragflow.jsonl"
+
+# 5. Tier 2: generate answers on each arm, then score with RAGAS.
+uv run python -m evals.benchmark answer --queries "$RUN/queries.jsonl" \
+  --ground-truth "$RUN/references.json" --base-url http://127.0.0.1:8080 \
+  --arm lilbee --out "$RUN/answers-lilbee.jsonl"
+uv run python -m evals.benchmark score-ragas \
+  --samples "$RUN/answers-lilbee.jsonl" --out "$RUN/results.jsonl"
+
+# 6. Paired statistics (CI + p) into the same results file.
+uv run python -m evals.benchmark stats --manifest "$RUN/manifest.frozen.json" \
+  --metrics-a "$RUN/ir-lilbee.jsonl" --metrics-b "$RUN/ir-ragflow.jsonl" \
+  --arm-a-label lilbee --arm-b-label ragflow --out "$RUN/results.jsonl"
+
+# 7. Render the report, then power the pod off.
+uv run python -m evals.benchmark report \
+  --results "$RUN/results.jsonl" --out "$RUN/report.md"
+```
+
+Keep the frozen manifest, run files, and qrels: with those, a third party can
+re-score the whole Tier-1 comparison without the pod.
+
+## Optional dependencies
+
+The heavy scorers and dataset loaders are imported lazily, so the harness (and
+its tests) load without them. They are kept out of the shipped lock on purpose
+(installing them would otherwise drag core dependencies backward), so install
+them from the standalone requirements file on the benchmark pod:
+
+```bash
+uv pip install -r evals/benchmark/requirements.txt   # pytrec_eval, ragas, beir, datasets
+```
+
+## Determinism and resume
+
+- The frozen manifest pins the bootstrap seed, resamples, and alpha, so every
+  CI is reproducible.
+- `collect` and `answer` checkpoint per query. Kill and re-run freely; only
+  unfinished queries repeat, and the run file is rebuilt from the full
+  checkpoint each time.

evals/benchmark/RESULTS.md

@@ -0,0 +1,107 @@
+# Retrieval benchmark results — Qwen3-Embedding-0.6B
+
+A significance-tested measurement of lilbee's hybrid retrieval against its own
+vector-only baseline on three public, human-labeled BEIR datasets. The point of
+this run is to answer two questions with evidence, not opinion: does lilbee's
+rank fusion actually improve retrieval over the raw embedder, and is a single
+fusion weight the right design.
+
+## Method
+
+- **Embedder:** Qwen3-Embedding-0.6B (GGUF, Q8_0), served once; every arm points
+  at the same model, so retrieval is the only variable.
+- **Datasets:** BEIR SciFact, NFCorpus, FiQA, using their published TREC qrels
+  as-is. No labels were derived or altered.
+- **Metrics:** nDCG@10, Recall@20, MRR@10, scored with `pytrec_eval` against the
+  native qrels. Nothing is graded by a model, so the numbers are reproducible by
+  anyone with the run files and qrels.
+- **Arms:** `dense` is vector-only (the lexical and title arms silenced);
+  `w=X` is hybrid fusion with the BM25 arm at `lexical_fusion_weight=X` and the
+  title arm on. This isolates what fusion adds over the embedder alone.
+- **Significance:** every hybrid-vs-dense difference is paired per query and gets
+  a bootstrap 95% CI and a randomization-test p-value. A difference whose CI
+  crosses zero is reported as not significant, not as a win.
+- **Condition:** query expansion was off (no chat model served), so these are
+  pure first-pass retrieval numbers with no LLM query rewriting on either arm.
+
+## Results (nDCG@10)
+
+| Dataset | dense | w=1.0 | w=0.75 | w=0.5 | w=0.25 |
+|---|---|---|---|---|---|
+| NFCorpus (n=322) | 0.3666 | 0.3734 | 0.3743 | 0.3756 | **0.3767** |
+| SciFact (n=300) | 0.6981 | **0.7280** | 0.7226 | 0.7167 | 0.7155 |
+| FiQA (n=648) | **0.4598** | 0.4033 | 0.4289 | 0.4333 | 0.4363 |
+
+## Significance (best hybrid weight vs. dense, nDCG@10)
+
+| Dataset | best hybrid | Δ vs dense | 95% CI | p | verdict |
+|---|---|---|---|---|---|
+| NFCorpus | w=0.25 | **+0.0100** | [+0.0022, +0.0180] | 0.012 | significant win |
+| SciFact | w=1.0 | **+0.0299** | [+0.0073, +0.0528] | 0.011 | significant win |
+| FiQA | w=0.25 | **−0.0235** | [−0.0356, −0.0118] | 0.0004 | significant regression |
+
+On FiQA every hybrid weight loses to dense (w=1.0 is worst at −0.0564, p=0.0001);
+lowering the weight reduces the damage but never recovers it. On NFCorpus the
+win is only significant at lower weights (w=1.0 is +0.0068, p=0.21, not
+significant). On SciFact every weight wins, and higher is better.
+
+## What this establishes
+
+1. **Hybrid fusion adds real, significant value on two of three corpora** — once
+   it actually runs (see the bug below). This is the raw embedder plus lilbee's
+   fusion beating the raw embedder alone, measured, not asserted.
+2. **No single fixed fusion weight wins everywhere.** SciFact wants a strong
+   lexical arm (w=1.0), NFCorpus wants a weak one (w=0.25), and FiQA wants none.
+   The optimal weight is corpus-dependent, which is the empirical case for
+   gating the lexical weight per query rather than fixing it — the adaptive
+   fusion this run motivates.
+
+## The bug this run caught
+
+Before any of these numbers meant anything, every hybrid config scored
+identically to the vector-only baseline. The cause was a production defect, not
+a benchmark artifact: `ensure_fts_index()` runs `table.optimize()` on an
+existing index, which on a real-sized corpus hits a LanceDB encoding bug and
+raises; the failure was swallowed and left hybrid search disabled, so every
+query silently fell back to vector-only. It only surfaced on real data — a
+tiny local corpus never tripped the LanceDB bug. Fixed by marking an existing,
+queryable index ready before the best-effort optimize, so an optimize failure
+degrades to a warning instead of disabling retrieval corpus-wide.
+
+## Reproducing
+
+The frozen manifest, the TREC run files, and the qrels are kept under
+`results/`. With those, anyone can re-score Tier 1 without a GPU:
+
+```bash
+gunzip -k results/scifact/run-w1.0.trec.gz
+python -m evals.benchmark score-ir --qrels results/scifact/qrels.json \
+  --run results/scifact/run-w1.0.trec --dataset scifact --run-tag w1.0 --out /tmp/ir.jsonl
+```
+
+## Adaptive fusion
+
+Because no fixed weight wins everywhere, a follow-up run measured `adaptive_fusion`
+(the BM25 weight scaled per query by how peaked the vector ranking is) at three
+confidence margins, on the same three datasets. Best margin was 0.15.
+
+| Dataset | dense | w=1.0 | adaptive (m=0.15) | adaptive vs dense |
+|---|---|---|---|---|
+| NFCorpus | 0.3666 | 0.3732 | 0.3751 | +0.0085, p=0.033, **sig win** |
+| SciFact | 0.6981 | 0.7280 | 0.7186 | +0.0205, p=0.009, **sig win** |
+| FiQA | 0.4593 | 0.4027 | 0.4410 | −0.0184, p=0.003, sig regression |
+
+Adaptive fusion is the best single policy found. It beats the fixed w=1.0 default
+on all three datasets, keeps the significant NFCorpus and SciFact wins, and gives
+the smallest FiQA regression of any config (−0.018 vs −0.056 at w=1.0 and −0.023
+at the best fixed weight). It does not fully erase FiQA's regression — pure dense
+still wins there — so the vector-margin confidence signal reduces, but does not
+eliminate, the cases where the lexical arm hurts. Run files and stats are under
+`results-adaptive/`.
+
+## Scope
+
+This is the 0.6B-embedder study: it validates the fusion mechanism and the
+FTS-optimize fix, and it sizes the corpus-dependence of the lexical weight. It
+is deliberately not a headline "lilbee vs. everyone" number — a larger embedder
+and the RAGFlow A/B arm are separate runs.

evals/benchmark/__init__.py

@@ -0,0 +1,9 @@
+"""Preregistered, reproducible A/B benchmark: lilbee vs RAGFlow retrieval.
+
+Two arms answer the same public, human-labeled datasets with the same served
+model, so retrieval is the only variable. Tier 1 scores ranked results against
+the datasets' own relevance labels with pytrec_eval (no model opinion); Tier 2
+layers RAGAS answer-quality metrics on top, corroborated by the blind judge
+from ``evals.retrieval``. Lives outside ``src/`` on purpose: it never ships in
+the package.
+"""

evals/benchmark/__main__.py

@@ -0,0 +1,6 @@
+"""Runs the benchmark CLI: python -m evals.benchmark <subcommand>."""
+
+from evals.benchmark.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

evals/benchmark/bootstrap_ragflow.py

@@ -0,0 +1,141 @@
+"""Bring up a RAGFlow arm bound to the shared model, ready for collection.
+
+Given a running RAGFlow, this registers the shared OpenAI-compatible LLM and
+embedding endpoint (so generation is identical to lilbee's), creates a DeepDoc
+dataset, uploads a corpus directory, waits for parsing, creates a chat
+assistant bound to the dataset, and prints the api_key and assistant_id the
+collector and answer steps consume.
+
+This is a live operational script: it only talks to a real RAGFlow and is never
+imported by the test suite. Run it on the pod once RAGFlow's stack is up.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+DATASETS_ROUTE = "/api/v1/datasets"
+DOCUMENTS_ROUTE = "/api/v1/datasets/{dataset_id}/documents"
+PARSE_ROUTE = "/api/v1/datasets/{dataset_id}/chunks"
+CHATS_ROUTE = "/api/v1/chats"
+PARSE_POLL_SECONDS = 10.0
+PARSE_MAX_POLLS = 360
+REQUEST_TIMEOUT_SECONDS = 120.0
+DONE_RATIO = 1.0
+
+
+def _client(base_url: str, api_key: str) -> httpx.Client:
+    return httpx.Client(
+        base_url=base_url.rstrip("/"),
+        headers={"Authorization": f"Bearer {api_key}"},
+        timeout=REQUEST_TIMEOUT_SECONDS,
+    )
+
+
+def _data(response: httpx.Response) -> Any:
+    response.raise_for_status()
+    body = response.json()
+    if body.get("code", 0) != 0:
+        raise RuntimeError(f"ragflow error: {body.get('message', body)}")
+    return body.get("data")
+
+
+def create_dataset(client: httpx.Client, name: str, embedding_model: str) -> str:
+    """Create a DeepDoc dataset and return its id."""
+    data = _data(
+        client.post(
+            DATASETS_ROUTE,
+            json={
+                "name": name,
+                "chunk_method": "naive",
+                "embedding_model": embedding_model,
+                "parser_config": {"layout_recognize": "DeepDOC"},
+            },
+        )
+    )
+    return str(data["id"])
+
+
+def upload_corpus(client: httpx.Client, dataset_id: str, corpus_dir: Path) -> list[str]:
+    """Upload every file under ``corpus_dir`` and return the document ids."""
+    route = DOCUMENTS_ROUTE.format(dataset_id=dataset_id)
+    files = [("file", (path.name, path.read_bytes())) for path in sorted(corpus_dir.iterdir())]
+    data = _data(client.post(route, files=files))
+    return [str(doc["id"]) for doc in data]
+
+
+def parse_and_wait(client: httpx.Client, dataset_id: str, document_ids: list[str]) -> None:
+    """Trigger parsing and block until every document finishes."""
+    route = PARSE_ROUTE.format(dataset_id=dataset_id)
+    _data(client.post(route, json={"document_ids": document_ids}))
+    docs_route = DOCUMENTS_ROUTE.format(dataset_id=dataset_id)
+    for _ in range(PARSE_MAX_POLLS):
+        docs = _data(client.get(docs_route))["docs"]
+        if all(doc.get("progress", 0.0) >= DONE_RATIO for doc in docs):
+            return
+        time.sleep(PARSE_POLL_SECONDS)
+    raise RuntimeError("ragflow parsing did not finish within the poll budget")
+
+
+def create_assistant(client: httpx.Client, name: str, dataset_id: str, llm_model: str) -> str:
+    """Create a chat assistant bound to the dataset and return its id."""
+    data = _data(
+        client.post(
+            CHATS_ROUTE,
+            json={
+                "name": name,
+                "dataset_ids": [dataset_id],
+                "llm": {"model_name": llm_model, "temperature": 0.0},
+            },
+        )
+    )
+    return str(data["id"])
+
+
+def bootstrap(args: argparse.Namespace) -> int:
+    client = _client(args.base_url, args.api_key)
+    dataset_id = create_dataset(client, args.dataset_name, args.embedding_model)
+    document_ids = upload_corpus(client, dataset_id, args.corpus_dir)
+    parse_and_wait(client, dataset_id, document_ids)
+    assistant_id = create_assistant(client, args.assistant_name, dataset_id, args.llm_model)
+    print(f"api_key={args.api_key}")
+    print(f"dataset_id={dataset_id}")
+    print(f"assistant_id={assistant_id}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base-url", required=True, help="RAGFlow base URL")
+    parser.add_argument("--api-key", required=True, help="RAGFlow API key")
+    parser.add_argument("--corpus-dir", type=Path, required=True)
+    parser.add_argument("--dataset-name", default="benchmark-corpus")
+    parser.add_argument("--assistant-name", default="benchmark-assistant")
+    parser.add_argument(
+        "--llm-model",
+        required=True,
+        help="the shared OpenAI-compatible generator, e.g. qwen2.5-72b-instruct",
+    )
+    parser.add_argument(
+        "--embedding-model", required=True, help="the shared embedder, e.g. qwen3-embedding"
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        return bootstrap(args)
+    except (httpx.HTTPError, RuntimeError, OSError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())