Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 26 additions & 12 deletions ingest/index_definitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -441,11 +441,30 @@ def index_definitions_for_paper(



def index_definitions_for_corpus(
lancedb_path: str | Path, paper_ids: list[str] | None = None,
) -> tuple[int, int]:
"""Index ``paper_ids``, or every paper in the chunks table when None.

Returns ``(rows written, papers indexed)``. A paper with no preamble
contributes zero rows and is not an error.
"""
if paper_ids is None:
import lancedb

chunks = lancedb.connect(str(lancedb_path)).open_table("chunks")
column = (
chunks.search().select(["paper_id"]).limit(chunks.count_rows())
.to_arrow().column("paper_id")
)
paper_ids = sorted(set(column.to_pylist()))
total = sum(index_definitions_for_paper(pid, lancedb_path) for pid in paper_ids)
return total, len(paper_ids)


def _cli(argv: list[str]) -> int:
import argparse

import lancedb

from ingest.store import DEFAULT_LANCEDB_PATH

parser = argparse.ArgumentParser(
Expand All @@ -461,16 +480,10 @@ def _cli(argv: list[str]) -> int:
help="papers to index (default: every paper in the chunks table)",
)
args = parser.parse_args(argv)
paper_ids = args.paper_ids
if not paper_ids:
chunks = lancedb.connect(args.lancedb_path).open_table("chunks")
column = (
chunks.search().select(["paper_id"]).limit(chunks.count_rows())
.to_arrow().column("paper_id")
)
paper_ids = sorted(set(column.to_pylist()))
total = sum(index_definitions_for_paper(pid, args.lancedb_path) for pid in paper_ids)
print(f"indexed {total} definition(s) across {len(paper_ids)} paper(s)")
total, papers = index_definitions_for_corpus(
args.lancedb_path, args.paper_ids or None
)
print(f"indexed {total} definition(s) across {papers} paper(s)")
return 0


Expand All @@ -481,6 +494,7 @@ def _cli(argv: list[str]) -> int:

__all__ = [
"build_definitions_for_paper",
"index_definitions_for_corpus",
"index_definitions_for_paper",
"open_or_create_definitions_table",
"parse_macro_line",
Expand Down
73 changes: 73 additions & 0 deletions tests/test_notebook_ingest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
"""``tools/notebook_ingest.py`` builds the indexes a notebook is served from."""

from __future__ import annotations

import json
from types import SimpleNamespace

import lancedb
import pyarrow as pa

from ingest.preamble import PreambleDoc
from ingest.schema import CHUNKS_SCHEMA_V1
from ingest.store import write_corpus_version_marker
from tools import _notebook_common, notebook_ingest

PAPERS = ("2401.00001", "2401.00002")


def _chunk(paper_id: str) -> dict:
row = {field.name: None for field in CHUNKS_SCHEMA_V1}
row.update(
chunk_id=f"{paper_id}#c0", paper_id=paper_id, kind="theorem",
section_path=["1"], body_text="Let X be a scheme.",
body_tokens="let x be a scheme", chunker_version="v",
embedder_version="v",
)
return row


def _fake_bulk_ingest(paper_ids, *, lancedb_staging_path, **_kwargs):
"""Stand in for the fetch/parse/chunk/embed loop: leave behind what a
successful run leaves, a chunks table and its version marker."""
rows = [_chunk(pid) for pid in paper_ids]
lancedb.connect(str(lancedb_staging_path)).create_table(
"chunks", data=pa.Table.from_pylist(rows, schema=CHUNKS_SCHEMA_V1)
)
write_corpus_version_marker(
lancedb_staging_path, version=1, chunker_version="v",
embedder_version="v", paper_count=len(rows), chunk_count=len(rows),
)
return SimpleNamespace(
papers_total=len(rows), papers_succeeded=len(rows),
papers_failed=0, ar5iv_hit_rate=1.0,
)


def test_ingest_builds_the_definitions_table(tmp_path, monkeypatch):
base = tmp_path / "notebooks"
nb_dir = base / "demo-nb"
nb_dir.mkdir(parents=True)
(nb_dir / "papers.txt").write_text("\n".join(PAPERS) + "\n", encoding="utf-8")
preamble_dir = tmp_path / "preamble"
for paper_id in PAPERS:
doc = PreambleDoc(
paper_id=paper_id, source_hash="0" * 64,
macros=[r"\newcommand{\RR}{\mathbb{R}}"],
preamble_text="", preamble_hash="0" * 16,
)
(preamble_dir / paper_id).mkdir(parents=True)
(preamble_dir / paper_id / "preamble.json").write_text(
json.dumps(doc.to_dict()), encoding="utf-8"
)
monkeypatch.setattr(_notebook_common, "NOTEBOOKS_BASE", base)
monkeypatch.setattr("ingest.preamble.PREAMBLE_DIR", preamble_dir)
monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_bulk_ingest)

assert notebook_ingest.run("demo-nb") == 0

definitions = (
lancedb.connect(str(nb_dir / "lancedb")).open_table("definitions").to_arrow()
)
assert sorted(definitions.column("paper_id").to_pylist()) == sorted(PAPERS)
assert set(definitions.column("symbol").to_pylist()) == {r"\RR"}
15 changes: 15 additions & 0 deletions tests/tools/test_notebook_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,9 @@ def _fake_ingest(paper_ids, **kw):
return _FakeSummary()
monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest)
monkeypatch.setattr(notebook_ingest, "build_bm25_index", lambda *a, **kw: None)
monkeypatch.setattr(
notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0)
)

rc = notebook_ingest.run("demo")
assert rc == 0
Expand Down Expand Up @@ -683,6 +686,9 @@ def _capture_build(*args, **kwargs):
captured_calls.append({"args": args, "kwargs": kwargs})

monkeypatch.setattr(notebook_ingest, "build_bm25_index", _capture_build)
monkeypatch.setattr(
notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0)
)

rc = notebook_ingest.run("myslug")
assert rc == 0
Expand Down Expand Up @@ -737,6 +743,9 @@ def _capture_root(*args, **kwargs):
roots_seen.append(str(kwargs.get("index_root", "")))

monkeypatch.setattr(notebook_ingest, "build_bm25_index", _capture_root)
monkeypatch.setattr(
notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0)
)
monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest_alpha)
notebook_ingest.run("alpha")

Expand Down Expand Up @@ -789,6 +798,9 @@ def _fake_build(*args, **kwargs):

# First: a dry run just to find the notebook's index_root
monkeypatch.setattr(notebook_ingest, "build_bm25_index", _fake_build)
monkeypatch.setattr(
notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0)
)
notebook_ingest.run("demo")
nb_bm25_root = actual_root_holder[0]
assert nb_bm25_root is not None
Expand All @@ -800,6 +812,9 @@ def _fake_build(*args, **kwargs):
# Second run: now 3 version dirs → triggers warning
actual_root_holder.clear()
monkeypatch.setattr(notebook_ingest, "build_bm25_index", _fake_build)
monkeypatch.setattr(
notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0)
)
rc = notebook_ingest.run("demo")
assert rc == 0
err = capsys.readouterr().err
Expand Down
7 changes: 7 additions & 0 deletions tools/notebook_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@

from ingest.bm25_indexer import build_bm25_index
from ingest.bulk_ingest import run_bulk_ingest
from ingest.index_definitions import index_definitions_for_corpus
from tools._notebook_common import (
NotebookError,
notebook_dir,
Expand Down Expand Up @@ -181,6 +182,12 @@ def run(slug: str) -> int:
f"(at {bm25_v_dir}/)"
)

# get_definitions reads this notebook's own dataset, so build its
# definitions table here too; without it the tool answers
# index_status="absent" for every freshly ingested notebook.
rows, papers = index_definitions_for_corpus(lancedb_path)
print(f"definitions indexed: {rows} across {papers} paper(s)")

# Warn if multiple v<N>/ directories exist under the per-notebook
# BM25 root. Helps operators prune stale indices after a
# corpus rebuild that incremented the version.
Expand Down
Loading