Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions ingest/index_definitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,45 @@ def index_definitions_for_paper(
return len(rows)



def _cli(argv: list[str]) -> int:
import argparse

import lancedb

from ingest.store import DEFAULT_LANCEDB_PATH

parser = argparse.ArgumentParser(
prog="python -m ingest.index_definitions",
description="Rebuild the definitions index from each paper's preamble.",
)
parser.add_argument(
"--lancedb-path", default=str(DEFAULT_LANCEDB_PATH),
help="LanceDB dataset holding the chunks table",
)
parser.add_argument(
"paper_ids", nargs="*",
help="papers to index (default: every paper in the chunks table)",
)
args = parser.parse_args(argv)
paper_ids = args.paper_ids
if not paper_ids:
chunks = lancedb.connect(args.lancedb_path).open_table("chunks")
column = (
chunks.search().select(["paper_id"]).limit(chunks.count_rows())
.to_arrow().column("paper_id")
)
paper_ids = sorted(set(column.to_pylist()))
total = sum(index_definitions_for_paper(pid, args.lancedb_path) for pid in paper_ids)
print(f"indexed {total} definition(s) across {len(paper_ids)} paper(s)")
return 0


if __name__ == "__main__": # pragma: no cover
import sys

sys.exit(_cli(sys.argv[1:]))

__all__ = [
"build_definitions_for_paper",
"index_definitions_for_paper",
Expand Down
42 changes: 40 additions & 2 deletions ingest/index_equations.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,18 @@ def open_or_create_equations_table(lancedb_path: str | Path) -> Any:
)


def index_equation_trees(lancedb_path: str | Path) -> dict[str, int]:
def index_equation_trees(
lancedb_path: str | Path, *, rebuild: bool = False,
) -> dict[str, int]:
"""Populate ``mathml_tree_json`` on every row that has a
``mathml`` payload and a NULL ``mathml_tree_json``.

With ``rebuild=True``, recompute the tree of every row with a
``mathml`` payload, including ones that already have a tree. That is
the fix after :data:`MATHML_TREE_NORMALIZER_VERSION` changes, since a
stored tree is only comparable to a query tree built by the same
normalizer.

Returns a counts dict ``{processed, updated, skipped, failed}``
so the caller (ops tooling or tests) can verify what landed.

Expand Down Expand Up @@ -116,7 +124,7 @@ def index_equation_trees(lancedb_path: str | Path) -> dict[str, int]:
if not mathml:
counts["skipped"] += 1
continue
if row.get("mathml_tree_json"):
if row.get("mathml_tree_json") and not rebuild:
counts["skipped"] += 1
continue
try:
Expand Down Expand Up @@ -164,6 +172,36 @@ def index_equation_trees(lancedb_path: str | Path) -> dict[str, int]:
return counts



def _cli(argv: list[str]) -> int:
import argparse

from ingest.store import DEFAULT_LANCEDB_PATH

parser = argparse.ArgumentParser(
prog="python -m ingest.index_equations",
description="Build MathML trees for the equations table.",
)
parser.add_argument(
"--lancedb-path", default=str(DEFAULT_LANCEDB_PATH),
help="LanceDB dataset holding the equations table",
)
parser.add_argument(
"--rebuild", action="store_true",
help="recompute every tree, not only missing ones (needed after "
"the tree normalizer changes)",
)
args = parser.parse_args(argv)
counts = index_equation_trees(args.lancedb_path, rebuild=args.rebuild)
print(" ".join(f"{k}={v}" for k, v in counts.items()))
return 0


if __name__ == "__main__": # pragma: no cover
import sys

sys.exit(_cli(sys.argv[1:]))

__all__ = [
"index_equation_trees",
"open_or_create_equations_table",
Expand Down
46 changes: 46 additions & 0 deletions ingest/index_theorem_names.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,52 @@ def default_db_path(lancedb_path: str | Path) -> Path:
)



def _cli(argv: list[str]) -> int:
import argparse
import asyncio

import lancedb

from ingest.store import DEFAULT_LANCEDB_PATH

parser = argparse.ArgumentParser(
prog="python -m ingest.index_theorem_names",
description="Rebuild the theorem-name index from the chunks table.",
)
parser.add_argument(
"--lancedb-path", default=str(DEFAULT_LANCEDB_PATH),
help="LanceDB dataset holding the chunks table",
)
parser.add_argument(
"--db-path", default=None,
help="SQLite index to write (default: sqlite/theorem_names.db "
"beside the LanceDB root, where the server reads it)",
)
args = parser.parse_args(argv)
db_path = Path(args.db_path) if args.db_path else default_db_path(args.lancedb_path)
chunks = lancedb.connect(args.lancedb_path).open_table("chunks")

async def run() -> dict[str, int]:
store = await TheoremNamesStore.open(db_path)
try:
return await index_theorem_names_all_papers(chunks, store)
finally:
await store.close()

counts = asyncio.run(run())
print(
f"indexed {sum(counts.values())} theorem name(s) across "
f"{len(counts)} paper(s) into {db_path}"
)
return 0


if __name__ == "__main__": # pragma: no cover
import sys

sys.exit(_cli(sys.argv[1:]))

__all__ = [
"DEFAULT_CONFIDENCE",
"build_rows_from_chunks_for_paper",
Expand Down
2 changes: 1 addition & 1 deletion server/retrieval/equations.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def looks_like_mathml(text: str) -> bool:
#: Bumped whenever the normalization below changes the tree a given
#: MathML string produces. Stored ``equations.mathml_tree_json`` built
#: under an older version is NOT comparable to a freshly-parsed query
#: tree and must be re-indexed (``python -m ingest.index_equations``).
#: tree and must be re-indexed (``python -m ingest.index_equations --rebuild``).
#:
#: v1 → v2 (retrieval-unlocks-m4): parallel-markup reduction, invisible
#: operators dropped, redundant single-child ``mrow`` collapsed, text
Expand Down
152 changes: 152 additions & 0 deletions tests/test_index_clis.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
"""The derived-index modules run as the commands the server tells
operators to run (``python -m ingest.index_*``)."""

from __future__ import annotations

import json
import sqlite3
import subprocess
import sys
from pathlib import Path
from unittest.mock import patch

import lancedb
import pyarrow as pa

from ingest import index_definitions
from ingest.index_equations import open_or_create_equations_table
from ingest.preamble import PreambleDoc
from ingest.schema import CHUNKS_SCHEMA_V1, EQUATIONS_SCHEMA_V1

REPO_ROOT = Path(__file__).resolve().parents[1]


def _run(module: str, *args: str) -> subprocess.CompletedProcess:
return subprocess.run(
[sys.executable, "-m", module, *args],
cwd=REPO_ROOT, capture_output=True, text=True, timeout=120, check=False,
)


def _chunk(paper_id: str, n: int, theorem_name: str | None) -> dict:
row = {field.name: None for field in CHUNKS_SCHEMA_V1}
row.update(
chunk_id=f"{paper_id}#c{n}",
paper_id=paper_id,
kind="theorem",
section_path=["1"],
theorem_name=theorem_name,
body_text="body",
body_tokens="body",
chunker_version="v",
embedder_version="v",
)
return row


def _chunks_db(tmp_path: Path) -> Path:
path = tmp_path / "index" / "lancedb"
rows = [
_chunk("2401.00001", 0, "Riemann-Roch theorem"),
_chunk("2401.00001", 1, None),
_chunk("2401.00002", 0, "Serre duality"),
]
lancedb.connect(str(path)).create_table(
"chunks", data=pa.Table.from_pylist(rows, schema=CHUNKS_SCHEMA_V1)
)
return path


def test_theorem_names_command_builds_the_index_the_server_reads(tmp_path):
lancedb_path = _chunks_db(tmp_path)

result = _run("ingest.index_theorem_names", "--lancedb-path", str(lancedb_path))

assert result.returncode == 0, result.stderr
assert "across 2 paper(s)" in result.stdout
db = tmp_path / "index" / "sqlite" / "theorem_names.db"
conn = sqlite3.connect(str(db))
try:
names = {r[0] for r in conn.execute("SELECT display_name FROM theorem_names")}
finally:
conn.close()
assert names == {"Riemann-Roch theorem", "Serre duality"}


def _equations_db(tmp_path: Path) -> Path:
path = tmp_path / "lancedb"
path.mkdir()
table = open_or_create_equations_table(path)
rows = []
for eq_id, tree in (("eq-new", None), ("eq-stale", "stale tree")):
row = {field.name: None for field in EQUATIONS_SCHEMA_V1}
row.update(
equation_id=eq_id, paper_id="2401.00001",
presentation_latex="x", mathml="<math><mi>x</mi></math>",
mathml_tree_json=tree,
)
rows.append(row)
table.add(pa.Table.from_pylist(rows, schema=EQUATIONS_SCHEMA_V1))
return path


def _trees(path: Path) -> dict[str, str]:
table = lancedb.connect(str(path)).open_table("equations").to_arrow()
return dict(zip(
table.column("equation_id").to_pylist(),
table.column("mathml_tree_json").to_pylist(),
strict=True,
))


def test_equations_command_fills_missing_trees_and_keeps_others(tmp_path):
path = _equations_db(tmp_path)

result = _run("ingest.index_equations", "--lancedb-path", str(path))

assert result.returncode == 0, result.stderr
trees = _trees(path)
assert json.loads(trees["eq-new"])
assert trees["eq-stale"] == "stale tree"


def test_equations_rebuild_replaces_stale_trees(tmp_path):
path = _equations_db(tmp_path)

result = _run("ingest.index_equations", "--lancedb-path", str(path), "--rebuild")

assert result.returncode == 0, result.stderr
trees = _trees(path)
assert trees["eq-stale"] == trees["eq-new"]
assert json.loads(trees["eq-stale"])


def test_definitions_command_is_runnable():
result = _run("ingest.index_definitions", "--help")

assert result.returncode == 0
assert "usage: python -m ingest.index_definitions" in result.stdout


def test_definitions_command_indexes_every_paper_in_the_chunks_table(
tmp_path, capsys
):
lancedb_path = _chunks_db(tmp_path)
preamble_dir = tmp_path / "preamble"
for paper_id in ("2401.00001", "2401.00002"):
doc = PreambleDoc(
paper_id=paper_id, source_hash="0" * 64,
macros=[r"\newcommand{\RR}{\mathbb{R}}"],
preamble_text="", preamble_hash="0" * 16,
)
(preamble_dir / paper_id).mkdir(parents=True)
(preamble_dir / paper_id / "preamble.json").write_text(
json.dumps(doc.to_dict()), encoding="utf-8"
)

with patch("ingest.preamble.PREAMBLE_DIR", preamble_dir):
assert index_definitions._cli(["--lancedb-path", str(lancedb_path)]) == 0

assert "indexed 2 definition(s) across 2 paper(s)" in capsys.readouterr().out
table = lancedb.connect(str(lancedb_path)).open_table("definitions").to_arrow()
assert sorted(table.column("paper_id").to_pylist()) == ["2401.00001", "2401.00002"]
Loading