From c2b39ec20d00dae9a4cbdaeeed25d824cf3533d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 15:53:17 +0000 Subject: [PATCH] feat(tools): build definitions during notebook ingest get_definitions reads a notebook's own LanceDB dataset, but notebook ingest built only BM25 there, so every freshly ingested notebook answered index_status="absent" until someone ran the indexer by hand. tools/notebook_ingest.py now builds the definitions table right after BM25, over every paper in the notebook's chunks table. The "every paper in the chunks table" logic moves out of the index_definitions CLI into index_definitions_for_corpus, which both the CLI and notebook ingest call. The other derived indexes are deliberately not built here: - Theorem names live in one global SQLite file (#300). Building it from a notebook would mix that notebook's chunk ids into the shared index. - Equation trees need an equations table, and no ingest path extracts equations yet. - The staging bulk-ingest path is unchanged. Cutover swaps only the LanceDB directory, so an index built beside staging (theorem names, or BM25 under a per-dataset version number) would change what the live server reads before cutover, and rollback would not undo it. A new test drives notebook_ingest.run end to end with bulk ingest faked to leave a real chunks table and marker. It checks that the definitions table holds both papers; the unfixed run leaves no table. The existing tests that stub the BM25 build now stub this step too. Refs #310. --- ingest/index_definitions.py | 38 ++++++++++----- tests/test_notebook_ingest.py | 73 ++++++++++++++++++++++++++++ tests/tools/test_notebook_scripts.py | 15 ++++++ tools/notebook_ingest.py | 7 +++ 4 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 tests/test_notebook_ingest.py diff --git a/ingest/index_definitions.py b/ingest/index_definitions.py index 33b7ff46..91715306 100644 --- a/ingest/index_definitions.py +++ b/ingest/index_definitions.py @@ -441,11 +441,30 @@ def index_definitions_for_paper( +def index_definitions_for_corpus( + lancedb_path: str | Path, paper_ids: list[str] | None = None, +) -> tuple[int, int]: + """Index ``paper_ids``, or every paper in the chunks table when None. + + Returns ``(rows written, papers indexed)``. A paper with no preamble + contributes zero rows and is not an error. + """ + if paper_ids is None: + import lancedb + + chunks = lancedb.connect(str(lancedb_path)).open_table("chunks") + column = ( + chunks.search().select(["paper_id"]).limit(chunks.count_rows()) + .to_arrow().column("paper_id") + ) + paper_ids = sorted(set(column.to_pylist())) + total = sum(index_definitions_for_paper(pid, lancedb_path) for pid in paper_ids) + return total, len(paper_ids) + + def _cli(argv: list[str]) -> int: import argparse - import lancedb - from ingest.store import DEFAULT_LANCEDB_PATH parser = argparse.ArgumentParser( @@ -461,16 +480,10 @@ def _cli(argv: list[str]) -> int: help="papers to index (default: every paper in the chunks table)", ) args = parser.parse_args(argv) - paper_ids = args.paper_ids - if not paper_ids: - chunks = lancedb.connect(args.lancedb_path).open_table("chunks") - column = ( - chunks.search().select(["paper_id"]).limit(chunks.count_rows()) - .to_arrow().column("paper_id") - ) - paper_ids = sorted(set(column.to_pylist())) - total = sum(index_definitions_for_paper(pid, args.lancedb_path) for pid in paper_ids) - print(f"indexed {total} definition(s) across {len(paper_ids)} paper(s)") + total, papers = index_definitions_for_corpus( + args.lancedb_path, args.paper_ids or None + ) + print(f"indexed {total} definition(s) across {papers} paper(s)") return 0 @@ -481,6 +494,7 @@ def _cli(argv: list[str]) -> int: __all__ = [ "build_definitions_for_paper", + "index_definitions_for_corpus", "index_definitions_for_paper", "open_or_create_definitions_table", "parse_macro_line", diff --git a/tests/test_notebook_ingest.py b/tests/test_notebook_ingest.py new file mode 100644 index 00000000..7997c02f --- /dev/null +++ b/tests/test_notebook_ingest.py @@ -0,0 +1,73 @@ +"""``tools/notebook_ingest.py`` builds the indexes a notebook is served from.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import lancedb +import pyarrow as pa + +from ingest.preamble import PreambleDoc +from ingest.schema import CHUNKS_SCHEMA_V1 +from ingest.store import write_corpus_version_marker +from tools import _notebook_common, notebook_ingest + +PAPERS = ("2401.00001", "2401.00002") + + +def _chunk(paper_id: str) -> dict: + row = {field.name: None for field in CHUNKS_SCHEMA_V1} + row.update( + chunk_id=f"{paper_id}#c0", paper_id=paper_id, kind="theorem", + section_path=["1"], body_text="Let X be a scheme.", + body_tokens="let x be a scheme", chunker_version="v", + embedder_version="v", + ) + return row + + +def _fake_bulk_ingest(paper_ids, *, lancedb_staging_path, **_kwargs): + """Stand in for the fetch/parse/chunk/embed loop: leave behind what a + successful run leaves, a chunks table and its version marker.""" + rows = [_chunk(pid) for pid in paper_ids] + lancedb.connect(str(lancedb_staging_path)).create_table( + "chunks", data=pa.Table.from_pylist(rows, schema=CHUNKS_SCHEMA_V1) + ) + write_corpus_version_marker( + lancedb_staging_path, version=1, chunker_version="v", + embedder_version="v", paper_count=len(rows), chunk_count=len(rows), + ) + return SimpleNamespace( + papers_total=len(rows), papers_succeeded=len(rows), + papers_failed=0, ar5iv_hit_rate=1.0, + ) + + +def test_ingest_builds_the_definitions_table(tmp_path, monkeypatch): + base = tmp_path / "notebooks" + nb_dir = base / "demo-nb" + nb_dir.mkdir(parents=True) + (nb_dir / "papers.txt").write_text("\n".join(PAPERS) + "\n", encoding="utf-8") + preamble_dir = tmp_path / "preamble" + for paper_id in PAPERS: + doc = PreambleDoc( + paper_id=paper_id, source_hash="0" * 64, + macros=[r"\newcommand{\RR}{\mathbb{R}}"], + preamble_text="", preamble_hash="0" * 16, + ) + (preamble_dir / paper_id).mkdir(parents=True) + (preamble_dir / paper_id / "preamble.json").write_text( + json.dumps(doc.to_dict()), encoding="utf-8" + ) + monkeypatch.setattr(_notebook_common, "NOTEBOOKS_BASE", base) + monkeypatch.setattr("ingest.preamble.PREAMBLE_DIR", preamble_dir) + monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_bulk_ingest) + + assert notebook_ingest.run("demo-nb") == 0 + + definitions = ( + lancedb.connect(str(nb_dir / "lancedb")).open_table("definitions").to_arrow() + ) + assert sorted(definitions.column("paper_id").to_pylist()) == sorted(PAPERS) + assert set(definitions.column("symbol").to_pylist()) == {r"\RR"} diff --git a/tests/tools/test_notebook_scripts.py b/tests/tools/test_notebook_scripts.py index f40eeac6..87a8377a 100644 --- a/tests/tools/test_notebook_scripts.py +++ b/tests/tools/test_notebook_scripts.py @@ -406,6 +406,9 @@ def _fake_ingest(paper_ids, **kw): return _FakeSummary() monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest) monkeypatch.setattr(notebook_ingest, "build_bm25_index", lambda *a, **kw: None) + monkeypatch.setattr( + notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) + ) rc = notebook_ingest.run("demo") assert rc == 0 @@ -683,6 +686,9 @@ def _capture_build(*args, **kwargs): captured_calls.append({"args": args, "kwargs": kwargs}) monkeypatch.setattr(notebook_ingest, "build_bm25_index", _capture_build) + monkeypatch.setattr( + notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) + ) rc = notebook_ingest.run("myslug") assert rc == 0 @@ -737,6 +743,9 @@ def _capture_root(*args, **kwargs): roots_seen.append(str(kwargs.get("index_root", ""))) monkeypatch.setattr(notebook_ingest, "build_bm25_index", _capture_root) + monkeypatch.setattr( + notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) + ) monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest_alpha) notebook_ingest.run("alpha") @@ -789,6 +798,9 @@ def _fake_build(*args, **kwargs): # First: a dry run just to find the notebook's index_root monkeypatch.setattr(notebook_ingest, "build_bm25_index", _fake_build) + monkeypatch.setattr( + notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) + ) notebook_ingest.run("demo") nb_bm25_root = actual_root_holder[0] assert nb_bm25_root is not None @@ -800,6 +812,9 @@ def _fake_build(*args, **kwargs): # Second run: now 3 version dirs → triggers warning actual_root_holder.clear() monkeypatch.setattr(notebook_ingest, "build_bm25_index", _fake_build) + monkeypatch.setattr( + notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) + ) rc = notebook_ingest.run("demo") assert rc == 0 err = capsys.readouterr().err diff --git a/tools/notebook_ingest.py b/tools/notebook_ingest.py index 96a38130..98a4de32 100644 --- a/tools/notebook_ingest.py +++ b/tools/notebook_ingest.py @@ -43,6 +43,7 @@ from ingest.bm25_indexer import build_bm25_index from ingest.bulk_ingest import run_bulk_ingest +from ingest.index_definitions import index_definitions_for_corpus from tools._notebook_common import ( NotebookError, notebook_dir, @@ -181,6 +182,12 @@ def run(slug: str) -> int: f"(at {bm25_v_dir}/)" ) + # get_definitions reads this notebook's own dataset, so build its + # definitions table here too; without it the tool answers + # index_status="absent" for every freshly ingested notebook. + rows, papers = index_definitions_for_corpus(lancedb_path) + print(f"definitions indexed: {rows} across {papers} paper(s)") + # Warn if multiple v/ directories exist under the per-notebook # BM25 root. Helps operators prune stale indices after a # corpus rebuild that incremented the version.