From da8a702a96ed33a0ad1c16979a0427c3f8b159b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 23:08:29 +0000 Subject: [PATCH] feat(ingest): build derived indexes into staging The producers of a staged dataset wrote chunks only. Its BM25, definitions and theorem-name indexes were built later, if at all, so a cutover could promote chunks that nothing indexed. ingest.derived_indexes.build_derived_indexes builds all three inside the dataset for the version in its marker. run_bulk_ingest calls it once a paper has landed, run_re_embed before it marks the staging run complete, and run_delta before it advances its harvest state. Because the indexes now live in the dataset, ops/cutover.py and tools/notebook_cutover.py promote and roll them back with the chunks without knowing they exist. notebook_ingest drops its own index builds, which run_bulk_ingest now makes. python -m ingest.derived_indexes rebuilds them by hand. The equations table is still produced by extraction, not here. Closes #555 Closes #310 Closes #551 --- ingest/bulk_ingest.py | 5 + ingest/derived_indexes.py | 82 +++++++++++++ ingest/oai_delta.py | 8 ++ ingest/re_embed.py | 6 + tests/test_bulk_ingest.py | 38 ++++++ tests/test_derived_indexes.py | 116 ++++++++++++++++++ tests/test_notebook_ingest.py | 94 --------------- tests/test_oai_delta.py | 35 ++++++ tests/test_re_embed.py | 47 ++++++++ tests/tools/test_notebook_scripts.py | 170 ++------------------------- tools/notebook_ingest.py | 49 ++------ 11 files changed, 357 insertions(+), 293 deletions(-) create mode 100644 ingest/derived_indexes.py create mode 100644 tests/test_derived_indexes.py delete mode 100644 tests/test_notebook_ingest.py diff --git a/ingest/bulk_ingest.py b/ingest/bulk_ingest.py index bea09387..99c92fbc 100644 --- a/ingest/bulk_ingest.py +++ b/ingest/bulk_ingest.py @@ -63,6 +63,7 @@ try_cache, ) from ingest.chunker import STRUCTURE_SIGNAL_CLASSES, chunk_paper +from ingest.derived_indexes import build_derived_indexes from ingest.embedder import embed_paper from ingest.identifiers import is_valid_arxiv_paper_id from ingest.store import ( @@ -441,6 +442,10 @@ def run_bulk_ingest( if n % progress_interval == 0 or n == len(work): _log_progress(log_path, summary, paper_id) + # The dataset is not ready to promote until it carries its indexes. + if summary.papers_succeeded: + build_derived_indexes(lancedb_staging_path) + summary.elapsed_seconds = time.monotonic() - started # corpus-integrity-observability-e3: write the ingest-summary.json diff --git a/ingest/derived_indexes.py b/ingest/derived_indexes.py new file mode 100644 index 00000000..ef765764 --- /dev/null +++ b/ingest/derived_indexes.py @@ -0,0 +1,82 @@ +"""Build every index derived from a LanceDB dataset, inside that dataset. + +A dataset that carries its own indexes can be promoted, rolled back or +backed up as one directory. The producers of a dataset (bulk ingest, +re-embed) call :func:`build_derived_indexes` before they report the +dataset ready, so a cutover never promotes chunks without the indexes +that answer queries over them. + +The equations table is not built here: it comes from equation +extraction, which runs per paper during parsing. +""" + +from __future__ import annotations + +import logging +import sys +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True, slots=True) +class DerivedIndexSummary: + corpus_version: int + definitions: int + definition_papers: int + theorem_names: int + theorem_name_papers: int + + +def build_derived_indexes(lancedb_path: str | Path) -> DerivedIndexSummary: + """Build the BM25, definitions and theorem-name indexes of a dataset. + + Each is built for the version in the dataset's ``corpus-version.json`` + and written inside the dataset. Rebuilding is safe: BM25 skips a + version it already holds, and the other two replace their rows. + """ + from ingest.bm25_indexer import build_bm25_index + from ingest.index_definitions import index_definitions_for_corpus + from ingest.index_theorem_names import build_theorem_names_index + from server.corpus import read_corpus_version + + path = Path(lancedb_path) + info = read_corpus_version(lancedb_path=path) + if info is None: + raise FileNotFoundError(f"no corpus-version.json in {path}") + build_bm25_index(path, corpus_version=info.version) + definitions, definition_papers = index_definitions_for_corpus(path) + names = build_theorem_names_index(path) + summary = DerivedIndexSummary( + corpus_version=info.version, + definitions=definitions, + definition_papers=definition_papers, + theorem_names=sum(names.values()), + theorem_name_papers=len(names), + ) + logger.info("derived indexes built for %s: %s", path, summary) + return summary + + +def _cli(argv: list[str]) -> int: + import argparse + + parser = argparse.ArgumentParser( + description="Rebuild the indexes stored inside a LanceDB dataset." + ) + parser.add_argument("lancedb_path", type=Path) + args = parser.parse_args(argv) + s = build_derived_indexes(args.lancedb_path) + print( + f"corpus_version={s.corpus_version} " + f"definitions={s.definitions} ({s.definition_papers} papers) " + f"theorem_names={s.theorem_names} ({s.theorem_name_papers} papers)" + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(_cli(sys.argv[1:])) + +__all__ = ["DerivedIndexSummary", "build_derived_indexes"] diff --git a/ingest/oai_delta.py b/ingest/oai_delta.py index e171db24..1206b0ed 100644 --- a/ingest/oai_delta.py +++ b/ingest/oai_delta.py @@ -73,7 +73,9 @@ _log_parser_failure, ingest_one_paper, ) +from ingest.derived_indexes import build_derived_indexes from ingest.identifiers import is_valid_arxiv_paper_id +from ingest.store import CORPUS_VERSION_MARKER_NAME logger = logging.getLogger(__name__) @@ -816,6 +818,12 @@ def run_delta( else: summary.records_failed += 1 + # The staged dataset changed: rebuild its indexes before the run counts + # as done, so a cutover never promotes chunks without them. + changed = summary.records_ingested or summary.records_deleted + if changed and (lancedb_staging_path / CORPUS_VERSION_MARKER_NAME).is_file(): + build_derived_indexes(lancedb_staging_path) + summary.elapsed_seconds = time.monotonic() - started summary.budget_breached = summary.elapsed_seconds > budget_seconds if summary.budget_breached: diff --git a/ingest/re_embed.py b/ingest/re_embed.py index 22af84ab..d534c87c 100644 --- a/ingest/re_embed.py +++ b/ingest/re_embed.py @@ -49,6 +49,7 @@ from ingest.bulk_ingest import DEFAULT_LANCEDB_STAGING_PATH from ingest.chunker import chunk_paper from ingest.chunker_types import CHUNKER_VERSION, ChunkRecord +from ingest.derived_indexes import build_derived_indexes from ingest.embedder import EMBEDDER_VERSION, embed_paper from ingest.identifiers import is_valid_arxiv_paper_id, paper_id_from_chunk_id from ingest.schema import ( @@ -792,6 +793,11 @@ def run_re_embed( }, ) + # Before the terminal sentinel, so a staging dataset marked complete + # always carries the indexes built from its own chunks. + if not dry_run and (staging_lancedb_path / CORPUS_VERSION_MARKER_NAME).is_file(): + build_derived_indexes(staging_lancedb_path) + summary.elapsed_seconds = time.monotonic() - started # Finalize state on success (no failures past the per-paper isolation). diff --git a/tests/test_bulk_ingest.py b/tests/test_bulk_ingest.py index aa40d17f..56162b69 100644 --- a/tests/test_bulk_ingest.py +++ b/tests/test_bulk_ingest.py @@ -29,6 +29,17 @@ run_bulk_ingest, ) + +@pytest.fixture(autouse=True) +def derived_builds(monkeypatch) -> list: + """Record index builds: the outcomes here are mocks with no dataset.""" + calls: list = [] + monkeypatch.setattr( + "ingest.bulk_ingest.build_derived_indexes", lambda path: calls.append(path) + ) + return calls + + # --------------------------------------------------------------------------- # _read_paper_ids # --------------------------------------------------------------------------- @@ -165,6 +176,33 @@ def test_all_papers_succeed(self, tmp_path): assert summary.ar5iv_hit_rate == 1.0 assert not failures.exists() # no failures, no JSONL line + def test_builds_the_indexes_into_the_dataset_it_wrote( + self, tmp_path, derived_builds + ): + staging = tmp_path / "staging" + with patch( + "ingest.bulk_ingest.ingest_one_paper", + side_effect=[_fail_outcome("2401.00001"), _ok_outcome("2401.00002")], + ): + run_bulk_ingest( + ["2401.00001", "2401.00002"], lancedb_staging_path=staging, + failures_path=tmp_path / "f.jsonl", log_path=tmp_path / "log", + ops_dir=tmp_path, + ) + assert derived_builds == [staging] + + def test_builds_nothing_when_no_paper_landed(self, tmp_path, derived_builds): + with patch( + "ingest.bulk_ingest.ingest_one_paper", + side_effect=[_fail_outcome("2401.00001")], + ): + run_bulk_ingest( + ["2401.00001"], lancedb_staging_path=tmp_path / "staging", + failures_path=tmp_path / "f.jsonl", log_path=tmp_path / "log", + ops_dir=tmp_path, + ) + assert derived_builds == [] + def test_writes_ingest_summary_sentinel(self, tmp_path): """corpus-integrity-observability-e3 F3: run_bulk_ingest writes the ingest-summary.json sentinel at end-of-run (AC2). The driver WIRING was diff --git a/tests/test_derived_indexes.py b/tests/test_derived_indexes.py new file mode 100644 index 00000000..319243c0 --- /dev/null +++ b/tests/test_derived_indexes.py @@ -0,0 +1,116 @@ +"""``ingest.derived_indexes`` builds a dataset's indexes inside it.""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import lancedb +import pyarrow as pa + +from ingest.derived_indexes import build_derived_indexes +from ingest.preamble import PreambleDoc +from ingest.schema import CHUNKS_SCHEMA_V1 +from ingest.store import write_corpus_version_marker + +PAPERS = ("2401.00001", "2401.00002") + + +def _chunk(paper_id: str) -> dict: + row = {field.name: None for field in CHUNKS_SCHEMA_V1} + row.update( + chunk_id=f"{paper_id}#c0", paper_id=paper_id, kind="theorem", + section_path=["1"], body_text="Let X be a scheme.", + body_tokens="let x be a scheme", chunker_version="v", + theorem_name=f"Lemma of {paper_id}", embedder_version="v", + ) + return row + + +def _dataset(tmp_path, monkeypatch, name="lancedb-staging", papers=PAPERS): + """A dataset (chunks + marker) and the preambles of its papers.""" + staging = tmp_path / name + rows = [_chunk(pid) for pid in papers] + table = lancedb.connect(str(staging)).create_table( + "chunks", data=pa.Table.from_pylist(rows, schema=CHUNKS_SCHEMA_V1) + ) + write_corpus_version_marker( + staging, version=table.version, chunker_version="v", embedder_version="v", + paper_count=len(rows), chunk_count=len(rows), + ) + preamble_dir = tmp_path / "preamble" + for paper_id in papers: + doc = PreambleDoc( + paper_id=paper_id, source_hash="0" * 64, + macros=[r"\newcommand{\RR}{\mathbb{R}}"], + preamble_text="", preamble_hash="0" * 16, + ) + (preamble_dir / paper_id).mkdir(parents=True, exist_ok=True) + (preamble_dir / paper_id / "preamble.json").write_text( + json.dumps(doc.to_dict()), encoding="utf-8" + ) + monkeypatch.setattr("ingest.preamble.PREAMBLE_DIR", preamble_dir) + return staging + + +def test_a_staged_dataset_carries_its_own_indexes(tmp_path, monkeypatch): + staging = _dataset(tmp_path, monkeypatch) + + summary = build_derived_indexes(staging) + + # Nothing lands beside the dataset but the build-stats log. + assert {p.name for p in tmp_path.iterdir()} == {"lancedb-staging", "preamble", "ops"} + bm25 = staging / "derived" / "bm25" / f"v{summary.corpus_version}" + assert sorted(json.loads((bm25 / "chunk_ids.json").read_text())) == [ + f"{pid}#c0" for pid in PAPERS + ] + definitions = ( + lancedb.connect(str(staging)).open_table("definitions").to_arrow() + ) + assert sorted(definitions.column("paper_id").to_pylist()) == sorted(PAPERS) + assert set(definitions.column("symbol").to_pylist()) == {r"\RR"} + conn = sqlite3.connect(str(staging / "derived" / "theorem_names.db")) + try: + names = {r[0] for r in conn.execute("SELECT display_name FROM theorem_names")} + finally: + conn.close() + assert names == {f"Lemma of {pid}" for pid in PAPERS} + + +def test_rebuilding_is_safe(tmp_path, monkeypatch): + staging = _dataset(tmp_path, monkeypatch) + first = build_derived_indexes(staging) + assert build_derived_indexes(staging) == first + + +def _snapshot(dataset: Path) -> dict[str, bytes]: + derived = dataset / "derived" + return { + str(f.relative_to(derived)): f.read_bytes() + for f in sorted(derived.rglob("*")) if f.is_file() + } + + +def test_cutover_then_rollback_restores_the_live_indexes(tmp_path, monkeypatch): + from ops.cutover import perform_directory_swap, perform_rollback + from server.config import Config + + live = _dataset(tmp_path, monkeypatch, "lancedb", PAPERS[:1]) + build_derived_indexes(live) + staging = _dataset(tmp_path, monkeypatch, "lancedb-staging", PAPERS) + build_derived_indexes(staging) + before = _snapshot(live) + staged = _snapshot(staging) + served = Config(lancedb_path=live) + + perform_directory_swap( + active_path=live, staging_path=staging, + rollback_path=tmp_path / "lancedb-prev", + ) + assert _snapshot(live) == staged + perform_rollback(active_path=live, rollback_path=tmp_path / "lancedb-prev") + + assert _snapshot(live) == before + assert served.theorem_names_db_path.read_bytes() == before["theorem_names.db"] + assert served.bm25_index_root.is_relative_to(live) diff --git a/tests/test_notebook_ingest.py b/tests/test_notebook_ingest.py deleted file mode 100644 index 6b594abf..00000000 --- a/tests/test_notebook_ingest.py +++ /dev/null @@ -1,94 +0,0 @@ -"""``tools/notebook_ingest.py`` builds the indexes a notebook is served from.""" - -from __future__ import annotations - -import json -import sqlite3 -from types import SimpleNamespace - -import lancedb -import pyarrow as pa - -from ingest.preamble import PreambleDoc -from ingest.schema import CHUNKS_SCHEMA_V1 -from ingest.store import write_corpus_version_marker -from tools import _notebook_common, notebook_ingest - -PAPERS = ("2401.00001", "2401.00002") - - -def _chunk(paper_id: str) -> dict: - row = {field.name: None for field in CHUNKS_SCHEMA_V1} - row.update( - chunk_id=f"{paper_id}#c0", paper_id=paper_id, kind="theorem", - section_path=["1"], body_text="Let X be a scheme.", - body_tokens="let x be a scheme", chunker_version="v", - theorem_name=f"Lemma of {paper_id}", - embedder_version="v", - ) - return row - - -def _fake_bulk_ingest(paper_ids, *, lancedb_staging_path, **_kwargs): - """Stand in for the fetch/parse/chunk/embed loop: leave behind what a - successful run leaves, a chunks table and its version marker.""" - rows = [_chunk(pid) for pid in paper_ids] - lancedb.connect(str(lancedb_staging_path)).create_table( - "chunks", data=pa.Table.from_pylist(rows, schema=CHUNKS_SCHEMA_V1) - ) - write_corpus_version_marker( - lancedb_staging_path, version=1, chunker_version="v", - embedder_version="v", paper_count=len(rows), chunk_count=len(rows), - ) - return SimpleNamespace( - papers_total=len(rows), papers_succeeded=len(rows), - papers_failed=0, ar5iv_hit_rate=1.0, - ) - - -def test_ingest_builds_the_definitions_table(tmp_path, monkeypatch): - base = tmp_path / "notebooks" - nb_dir = base / "demo-nb" - nb_dir.mkdir(parents=True) - (nb_dir / "papers.txt").write_text("\n".join(PAPERS) + "\n", encoding="utf-8") - preamble_dir = tmp_path / "preamble" - for paper_id in PAPERS: - doc = PreambleDoc( - paper_id=paper_id, source_hash="0" * 64, - macros=[r"\newcommand{\RR}{\mathbb{R}}"], - preamble_text="", preamble_hash="0" * 16, - ) - (preamble_dir / paper_id).mkdir(parents=True) - (preamble_dir / paper_id / "preamble.json").write_text( - json.dumps(doc.to_dict()), encoding="utf-8" - ) - monkeypatch.setattr(_notebook_common, "NOTEBOOKS_BASE", base) - monkeypatch.setattr("ingest.preamble.PREAMBLE_DIR", preamble_dir) - monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_bulk_ingest) - - assert notebook_ingest.run("demo-nb") == 0 - - definitions = ( - lancedb.connect(str(nb_dir / "lancedb")).open_table("definitions").to_arrow() - ) - assert sorted(definitions.column("paper_id").to_pylist()) == sorted(PAPERS) - assert set(definitions.column("symbol").to_pylist()) == {r"\RR"} - - -def test_ingest_builds_the_theorem_name_index(tmp_path, monkeypatch): - base = tmp_path / "notebooks" - nb_dir = base / "demo-nb" - nb_dir.mkdir(parents=True) - (nb_dir / "papers.txt").write_text("\n".join(PAPERS) + "\n", encoding="utf-8") - monkeypatch.setattr(_notebook_common, "NOTEBOOKS_BASE", base) - monkeypatch.setattr("ingest.preamble.PREAMBLE_DIR", tmp_path / "preamble") - monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_bulk_ingest) - - assert notebook_ingest.run("demo-nb") == 0 - - conn = sqlite3.connect(str(nb_dir / "lancedb" / "derived" / "theorem_names.db")) - try: - names = {r[0] for r in conn.execute("SELECT display_name FROM theorem_names")} - finally: - conn.close() - assert names == {f"Lemma of {pid}" for pid in PAPERS} diff --git a/tests/test_oai_delta.py b/tests/test_oai_delta.py index f36d6d61..dbde5f60 100644 --- a/tests/test_oai_delta.py +++ b/tests/test_oai_delta.py @@ -406,6 +406,41 @@ def test_end_to_end_with_mock_records(self, tmp_path): assert state["last_resumption_token"] is None assert state["last_run_paper_count"] == 2 + def test_rebuilds_the_staged_indexes_before_advancing_state( + self, tmp_path, monkeypatch + ): + staging = tmp_path / "lancedb-staging" + staging.mkdir() + (staging / "corpus-version.json").write_text("{}") + state_path = tmp_path / "state.json" + builds: list = [] + monkeypatch.setattr( + "ingest.oai_delta.build_derived_indexes", + lambda path: builds.append( + (path, _read_state(state_path).get("last_successful_run_utc")) + ), + ) + fetcher = _MockFetcher([_final_page(["2401.00001"])]) + with patch( + "ingest.oai_delta.ingest_one_paper", + side_effect=lambda pid, **kw: _ok_paper_outcome(pid), + ): + run_delta( + sets=("math:math:AG",), + from_date="2026-05-14", + until_date="2026-05-14", + lancedb_staging_path=staging, + state_path=state_path, + log_path=tmp_path / "delta.log", + failures_path=tmp_path / "delta.jsonl", + timeout_flag_path=tmp_path / "timeout.flag", + ops_dir=tmp_path, + fetch_page=fetcher, + sleep_between_pages=lambda _t: None, + ) + assert builds == [(staging, None)] + assert _read_state(state_path)["last_successful_run_utc"] + def test_writes_ingest_summary_sentinel(self, tmp_path): """corpus-integrity-observability-e3 F3: run_delta writes the ingest-summary.json sentinel at end-of-run (AC2).""" diff --git a/tests/test_re_embed.py b/tests/test_re_embed.py index 9aef846b..c258ccfa 100644 --- a/tests/test_re_embed.py +++ b/tests/test_re_embed.py @@ -900,6 +900,53 @@ def _capturing_write(chunks, embeddings, lancedb_path=None): assert final["target_embedder_version"] == EMBEDDER_VERSION assert "chunks_re_embedded" in final + def test_indexes_are_built_before_the_run_is_marked_complete( + self, tmp_path + ): + active = tmp_path / "lancedb" + staging = tmp_path / "lancedb-staging" + _write_active_marker(active) + sentinel_path = staging / "re-embed-progress.json" + builds: list[tuple] = [] + + def _write(chunks, embeddings, lancedb_path=None): + (lancedb_path / "corpus-version.json").write_text("{}") + return 1 + + def _build(path): + builds.append((path, json.loads(sentinel_path.read_text())["status"])) + + paper_id = "2401.00001" + new_chunks = [_make_chunk(paper_id, f"arxiv:{paper_id}:{0:016x}")] + with ( + patch("ingest.re_embed.chunk_paper", lambda _pid: new_chunks), + patch( + "ingest.re_embed._index_old_chunk_ids_by_paper", + lambda _p: {paper_id: set()}, + ), + patch("ingest.re_embed._build_old_rows_index", lambda _p: {}), + patch( + "ingest.re_embed._check_staging_embedder_versions", + lambda _p, _v: None, + ), + patch("ingest.re_embed.embed_paper", lambda _pid: _ok_embed_stats()), + patch( + "ingest.re_embed.load_embed_record", + lambda pid: _ok_full_record(pid, [c.chunk_id for c in new_chunks]), + ), + patch("ingest.re_embed.write_chunks", _write), + patch("ingest.re_embed.build_derived_indexes", _build), + ): + run_re_embed( + active_lancedb_path=active, + staging_lancedb_path=staging, + state_path=tmp_path / "state.json", + failures_path=tmp_path / "failures.jsonl", + ) + + assert builds == [(staging, "in_progress")] + assert json.loads(sentinel_path.read_text())["status"] == "complete" + def test_sentinel_not_written_on_dry_run(self, tmp_path): active = tmp_path / "lancedb" staging = tmp_path / "lancedb-staging" diff --git a/tests/tools/test_notebook_scripts.py b/tests/tools/test_notebook_scripts.py index 2c46a5a8..c3977e99 100644 --- a/tests/tools/test_notebook_scripts.py +++ b/tests/tools/test_notebook_scripts.py @@ -405,13 +405,6 @@ def _fake_ingest(paper_ids, **kw): marker.write_text(json.dumps({"version": 5})) return _FakeSummary() monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest) - monkeypatch.setattr(notebook_ingest, "build_bm25_index", lambda *a, **kw: None) - monkeypatch.setattr( - notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) - ) - monkeypatch.setattr( - notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {} - ) rc = notebook_ingest.run("demo") assert rc == 0 @@ -659,124 +652,12 @@ def test_purge_aborts_cleanly_on_eof( assert nb.exists() # NOT purged -def test_ingest_builds_bm25_under_per_notebook_root( - notebooks_base: Path, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """build_bm25_index writes into derived/bm25 inside the notebook's - dataset, where the server reads it and cutover moves it.""" - nb = _seed_notebook(notebooks_base, "myslug", ["2303.07061"]) - - class _FakeSummary: - papers_total = 1 - papers_succeeded = 1 - papers_failed = 0 - @property - def ar5iv_hit_rate(self): return 1.0 - - def _fake_ingest(paper_ids, **kw): - marker = nb / "lancedb" / "corpus-version.json" - marker.write_text(json.dumps({"version": 5})) - return _FakeSummary() - - monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest) - - captured_calls: list[dict] = [] - - def _capture_build(*args, **kwargs): - captured_calls.append({"args": args, "kwargs": kwargs}) - - monkeypatch.setattr(notebook_ingest, "build_bm25_index", _capture_build) - monkeypatch.setattr( - notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) - ) - monkeypatch.setattr( - notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {} - ) - - rc = notebook_ingest.run("myslug") - assert rc == 0 - assert len(captured_calls) == 1 - # index_root must be the per-notebook path, not the global root - call = captured_calls[0] - index_root = call["kwargs"].get("index_root") - assert index_root is not None, "index_root must be passed explicitly" - # F4: assert structurally on Path.parts (separator-agnostic — a str - # substring check with a hard-coded "/" fails on Windows). - index_root_parts = Path(index_root).parts - assert "myslug" in index_root_parts, ( - f"index_root should be under the notebook dir; got {index_root}" - ) - assert Path(index_root) == nb / "lancedb" / "derived" / "bm25", ( - f"index_root should be /lancedb/derived/bm25; got {index_root}" - ) - # Confirm no .notebook_slug sentinel is written (FM-7: sentinel removed) - assert not (index_root / "v5" / ".notebook_slug").exists() - - -def test_ingest_per_notebook_bm25_root_no_collision( - notebooks_base: Path, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """notebook-bm25-isolation-m1: two notebooks at version 1 get - different index_root values — no collision by construction.""" - nb_a = _seed_notebook(notebooks_base, "alpha", ["2303.07061"]) - nb_b = _seed_notebook(notebooks_base, "beta", ["2303.07062"]) - - class _FakeSummary: - papers_total = 1 - papers_succeeded = 1 - papers_failed = 0 - @property - def ar5iv_hit_rate(self): return 1.0 - - def _fake_ingest_alpha(paper_ids, lancedb_staging_path, **kw): - marker = nb_a / "lancedb" / "corpus-version.json" - marker.write_text(json.dumps({"version": 1})) - return _FakeSummary() - - def _fake_ingest_beta(paper_ids, lancedb_staging_path, **kw): - marker = nb_b / "lancedb" / "corpus-version.json" - marker.write_text(json.dumps({"version": 1})) - return _FakeSummary() - - roots_seen: list[str] = [] - - def _capture_root(*args, **kwargs): - roots_seen.append(str(kwargs.get("index_root", ""))) - - monkeypatch.setattr(notebook_ingest, "build_bm25_index", _capture_root) - monkeypatch.setattr( - notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) - ) - monkeypatch.setattr( - notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {} - ) - monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest_alpha) - notebook_ingest.run("alpha") - - monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest_beta) - notebook_ingest.run("beta") - - assert len(roots_seen) == 2 - assert roots_seen[0] != roots_seen[1], ( - f"Two notebooks at version 1 produced the SAME index_root: " - f"{roots_seen[0]!r} — collision not prevented!" - ) - assert "alpha" in roots_seen[0] - assert "beta" in roots_seen[1] - - def test_ingest_warns_about_stale_bm25_versions( notebooks_base: Path, - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture, ) -> None: - """notebook-bm25-isolation-m1: when multiple v/ dirs exist under - the per-notebook BM25 root after build, print a WARN message.""" + """Older BM25 version directories left in the dataset draw a warning.""" nb = _seed_notebook(notebooks_base, "demo", ["2303.07061"]) class _FakeSummary: @@ -787,53 +668,18 @@ class _FakeSummary: def ar5iv_hit_rate(self): return 1.0 def _fake_ingest(paper_ids, **kw): - marker = nb / "lancedb" / "corpus-version.json" - marker.write_text(json.dumps({"version": 7})) + (nb / "lancedb" / "corpus-version.json").write_text(json.dumps({"version": 7})) + for version in (1, 2, 7): + (nb / "lancedb" / "derived" / "bm25" / f"v{version}").mkdir( + parents=True, exist_ok=True + ) return _FakeSummary() monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest) - # Pre-seed stale version dirs under the per-notebook BM25 root so the - # warning fires. We must capture the index_root to know where to seed. - actual_root_holder: list = [] - - def _fake_build(*args, **kwargs): - actual_root_holder.append(kwargs.get("index_root")) - # Mimic the indexer: create the v7 dir so glob finds it - root = kwargs.get("index_root") - if root is not None: - (root / "v7").mkdir(parents=True, exist_ok=True) - - # First: a dry run just to find the notebook's index_root - monkeypatch.setattr(notebook_ingest, "build_bm25_index", _fake_build) - monkeypatch.setattr( - notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) - ) - monkeypatch.setattr( - notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {} - ) - notebook_ingest.run("demo") - nb_bm25_root = actual_root_holder[0] - assert nb_bm25_root is not None - - # Pre-seed older version dirs to trigger the multi-dir warning - (nb_bm25_root / "v1").mkdir(parents=True, exist_ok=True) - (nb_bm25_root / "v2").mkdir(parents=True, exist_ok=True) - - # Second run: now 3 version dirs → triggers warning - actual_root_holder.clear() - monkeypatch.setattr(notebook_ingest, "build_bm25_index", _fake_build) - monkeypatch.setattr( - notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) - ) - monkeypatch.setattr( - notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {} - ) - rc = notebook_ingest.run("demo") - assert rc == 0 + assert notebook_ingest.run("demo") == 0 err = capsys.readouterr().err - assert "WARN" in err - assert "BM25 version directories exist" in err + assert "3 BM25 version directories exist" in err # --------------------------------------------------------------------------- diff --git a/tools/notebook_ingest.py b/tools/notebook_ingest.py index 736f4329..aec5a34c 100644 --- a/tools/notebook_ingest.py +++ b/tools/notebook_ingest.py @@ -1,4 +1,4 @@ -"""Run bulk_ingest against a notebook's papers.txt, then build BM25. +"""Run bulk_ingest against a notebook's papers.txt. Thin wrapper that: @@ -6,9 +6,9 @@ ``lancedb_staging_path=var/arxmcp/notebooks//lancedb``. 2. Reads the resulting ``corpus-version.json`` to find the per-notebook corpus version. -3. Builds the BM25, definitions and theorem-name indexes for that - version inside the dataset, under ``lancedb/derived/``, so a cutover - or rollback moves them with the chunks. +3. ``run_bulk_ingest`` has already built the dataset's BM25, definitions + and theorem-name indexes inside it, so a cutover or rollback moves + them with the chunks. The brief's ``ARXMCP_LANCEDB_PATH=...`` wording was wrong — that's the SERVER's env var, not a bulk_ingest one. ``bulk_ingest`` uses the @@ -20,8 +20,8 @@ uv run python tools/notebook_ingest.py Exit codes: - 0 — ingest succeeded AND BM25 index was built (or both skipped idempotently) - 1 — slug validation failed, ingest had any paper failure, or BM25 build raised + 0 — every paper ingested and the dataset's indexes were built + 1 — slug validation failed, or any paper failed """ from __future__ import annotations @@ -31,10 +31,8 @@ import logging import sys -from ingest.bm25_indexer import build_bm25_index, resolve_bm25_root +from ingest.bm25_indexer import resolve_bm25_root from ingest.bulk_ingest import run_bulk_ingest -from ingest.index_definitions import index_definitions_for_corpus -from ingest.index_theorem_names import build_theorem_names_index from tools._notebook_common import ( NotebookError, notebook_dir, @@ -142,39 +140,16 @@ def run(slug: str) -> int: f"see {failures_path}", file=sys.stderr, ) - # We still try to build BM25 on the partial corpus; the index - # reflects whatever did land. Exit non-zero so the operator - # notices. + # The indexes cover whatever did land. Exit non-zero so the + # operator notices. if summary.papers_succeeded == 0: - print("no papers succeeded — skipping BM25 build", file=sys.stderr) + print("no papers succeeded — no indexes built", file=sys.stderr) return 1 + # run_bulk_ingest built the dataset's indexes inside it. corpus_version = _read_corpus_version(lancedb_path) - # The BM25 root sits inside the dataset, where the server reads it. bm25_root = resolve_bm25_root(lancedb_path) - bm25_v_dir = bm25_root / f"v{corpus_version}" - logger.info( - "building BM25 for slug=%s lancedb=%s corpus_version=%d root=%s", - slug, lancedb_path, corpus_version, bm25_root, - ) - build_bm25_index( - str(lancedb_path), corpus_version=corpus_version, index_root=bm25_root, - ) - print( - f"BM25 built for corpus_version={corpus_version} " - f"(at {bm25_v_dir}/)" - ) - - # get_definitions reads this notebook's own dataset, so build its - # definitions table here too; without it the tool answers - # index_status="absent" for every freshly ingested notebook. - rows, papers = index_definitions_for_corpus(lancedb_path) - print(f"definitions indexed: {rows} across {papers} paper(s)") - # find_lemma_by_name reads the theorem-name index inside the same dataset. - names = build_theorem_names_index(lancedb_path) - print( - f"theorem names indexed: {sum(names.values())} across {len(names)} paper(s)" - ) + print(f"indexes built for corpus_version={corpus_version} (in {lancedb_path})") # Warn if multiple v/ directories exist under the per-notebook # BM25 root. Helps operators prune stale indices after a