diff --git a/ingest/index_definitions.py b/ingest/index_definitions.py index 1f28a38f..33b7ff46 100644 --- a/ingest/index_definitions.py +++ b/ingest/index_definitions.py @@ -440,6 +440,45 @@ def index_definitions_for_paper( return len(rows) + +def _cli(argv: list[str]) -> int: + import argparse + + import lancedb + + from ingest.store import DEFAULT_LANCEDB_PATH + + parser = argparse.ArgumentParser( + prog="python -m ingest.index_definitions", + description="Rebuild the definitions index from each paper's preamble.", + ) + parser.add_argument( + "--lancedb-path", default=str(DEFAULT_LANCEDB_PATH), + help="LanceDB dataset holding the chunks table", + ) + parser.add_argument( + "paper_ids", nargs="*", + help="papers to index (default: every paper in the chunks table)", + ) + args = parser.parse_args(argv) + paper_ids = args.paper_ids + if not paper_ids: + chunks = lancedb.connect(args.lancedb_path).open_table("chunks") + column = ( + chunks.search().select(["paper_id"]).limit(chunks.count_rows()) + .to_arrow().column("paper_id") + ) + paper_ids = sorted(set(column.to_pylist())) + total = sum(index_definitions_for_paper(pid, args.lancedb_path) for pid in paper_ids) + print(f"indexed {total} definition(s) across {len(paper_ids)} paper(s)") + return 0 + + +if __name__ == "__main__": # pragma: no cover + import sys + + sys.exit(_cli(sys.argv[1:])) + __all__ = [ "build_definitions_for_paper", "index_definitions_for_paper", diff --git a/ingest/index_equations.py b/ingest/index_equations.py index 4e7466be..ec9d2e67 100644 --- a/ingest/index_equations.py +++ b/ingest/index_equations.py @@ -80,10 +80,18 @@ def open_or_create_equations_table(lancedb_path: str | Path) -> Any: ) -def index_equation_trees(lancedb_path: str | Path) -> dict[str, int]: +def index_equation_trees( + lancedb_path: str | Path, *, rebuild: bool = False, +) -> dict[str, int]: """Populate ``mathml_tree_json`` on every row that has a ``mathml`` payload and a NULL ``mathml_tree_json``. + With ``rebuild=True``, recompute the tree of every row with a + ``mathml`` payload, including ones that already have a tree. That is + the fix after :data:`MATHML_TREE_NORMALIZER_VERSION` changes, since a + stored tree is only comparable to a query tree built by the same + normalizer. + Returns a counts dict ``{processed, updated, skipped, failed}`` so the caller (ops tooling or tests) can verify what landed. @@ -116,7 +124,7 @@ def index_equation_trees(lancedb_path: str | Path) -> dict[str, int]: if not mathml: counts["skipped"] += 1 continue - if row.get("mathml_tree_json"): + if row.get("mathml_tree_json") and not rebuild: counts["skipped"] += 1 continue try: @@ -164,6 +172,36 @@ def index_equation_trees(lancedb_path: str | Path) -> dict[str, int]: return counts + +def _cli(argv: list[str]) -> int: + import argparse + + from ingest.store import DEFAULT_LANCEDB_PATH + + parser = argparse.ArgumentParser( + prog="python -m ingest.index_equations", + description="Build MathML trees for the equations table.", + ) + parser.add_argument( + "--lancedb-path", default=str(DEFAULT_LANCEDB_PATH), + help="LanceDB dataset holding the equations table", + ) + parser.add_argument( + "--rebuild", action="store_true", + help="recompute every tree, not only missing ones (needed after " + "the tree normalizer changes)", + ) + args = parser.parse_args(argv) + counts = index_equation_trees(args.lancedb_path, rebuild=args.rebuild) + print(" ".join(f"{k}={v}" for k, v in counts.items())) + return 0 + + +if __name__ == "__main__": # pragma: no cover + import sys + + sys.exit(_cli(sys.argv[1:])) + __all__ = [ "index_equation_trees", "open_or_create_equations_table", diff --git a/ingest/index_theorem_names.py b/ingest/index_theorem_names.py index f52efbff..4d212550 100644 --- a/ingest/index_theorem_names.py +++ b/ingest/index_theorem_names.py @@ -196,6 +196,52 @@ def default_db_path(lancedb_path: str | Path) -> Path: ) + +def _cli(argv: list[str]) -> int: + import argparse + import asyncio + + import lancedb + + from ingest.store import DEFAULT_LANCEDB_PATH + + parser = argparse.ArgumentParser( + prog="python -m ingest.index_theorem_names", + description="Rebuild the theorem-name index from the chunks table.", + ) + parser.add_argument( + "--lancedb-path", default=str(DEFAULT_LANCEDB_PATH), + help="LanceDB dataset holding the chunks table", + ) + parser.add_argument( + "--db-path", default=None, + help="SQLite index to write (default: sqlite/theorem_names.db " + "beside the LanceDB root, where the server reads it)", + ) + args = parser.parse_args(argv) + db_path = Path(args.db_path) if args.db_path else default_db_path(args.lancedb_path) + chunks = lancedb.connect(args.lancedb_path).open_table("chunks") + + async def run() -> dict[str, int]: + store = await TheoremNamesStore.open(db_path) + try: + return await index_theorem_names_all_papers(chunks, store) + finally: + await store.close() + + counts = asyncio.run(run()) + print( + f"indexed {sum(counts.values())} theorem name(s) across " + f"{len(counts)} paper(s) into {db_path}" + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover + import sys + + sys.exit(_cli(sys.argv[1:])) + __all__ = [ "DEFAULT_CONFIDENCE", "build_rows_from_chunks_for_paper", diff --git a/server/retrieval/equations.py b/server/retrieval/equations.py index 4594a9e1..0c1d50bc 100644 --- a/server/retrieval/equations.py +++ b/server/retrieval/equations.py @@ -83,7 +83,7 @@ def looks_like_mathml(text: str) -> bool: #: Bumped whenever the normalization below changes the tree a given #: MathML string produces. Stored ``equations.mathml_tree_json`` built #: under an older version is NOT comparable to a freshly-parsed query -#: tree and must be re-indexed (``python -m ingest.index_equations``). +#: tree and must be re-indexed (``python -m ingest.index_equations --rebuild``). #: #: v1 → v2 (retrieval-unlocks-m4): parallel-markup reduction, invisible #: operators dropped, redundant single-child ``mrow`` collapsed, text diff --git a/tests/test_index_clis.py b/tests/test_index_clis.py new file mode 100644 index 00000000..bbbf607f --- /dev/null +++ b/tests/test_index_clis.py @@ -0,0 +1,152 @@ +"""The derived-index modules run as the commands the server tells +operators to run (``python -m ingest.index_*``).""" + +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +import lancedb +import pyarrow as pa + +from ingest import index_definitions +from ingest.index_equations import open_or_create_equations_table +from ingest.preamble import PreambleDoc +from ingest.schema import CHUNKS_SCHEMA_V1, EQUATIONS_SCHEMA_V1 + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _run(module: str, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-m", module, *args], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=120, check=False, + ) + + +def _chunk(paper_id: str, n: int, theorem_name: str | None) -> dict: + row = {field.name: None for field in CHUNKS_SCHEMA_V1} + row.update( + chunk_id=f"{paper_id}#c{n}", + paper_id=paper_id, + kind="theorem", + section_path=["1"], + theorem_name=theorem_name, + body_text="body", + body_tokens="body", + chunker_version="v", + embedder_version="v", + ) + return row + + +def _chunks_db(tmp_path: Path) -> Path: + path = tmp_path / "index" / "lancedb" + rows = [ + _chunk("2401.00001", 0, "Riemann-Roch theorem"), + _chunk("2401.00001", 1, None), + _chunk("2401.00002", 0, "Serre duality"), + ] + lancedb.connect(str(path)).create_table( + "chunks", data=pa.Table.from_pylist(rows, schema=CHUNKS_SCHEMA_V1) + ) + return path + + +def test_theorem_names_command_builds_the_index_the_server_reads(tmp_path): + lancedb_path = _chunks_db(tmp_path) + + result = _run("ingest.index_theorem_names", "--lancedb-path", str(lancedb_path)) + + assert result.returncode == 0, result.stderr + assert "across 2 paper(s)" in result.stdout + db = tmp_path / "index" / "sqlite" / "theorem_names.db" + conn = sqlite3.connect(str(db)) + try: + names = {r[0] for r in conn.execute("SELECT display_name FROM theorem_names")} + finally: + conn.close() + assert names == {"Riemann-Roch theorem", "Serre duality"} + + +def _equations_db(tmp_path: Path) -> Path: + path = tmp_path / "lancedb" + path.mkdir() + table = open_or_create_equations_table(path) + rows = [] + for eq_id, tree in (("eq-new", None), ("eq-stale", "stale tree")): + row = {field.name: None for field in EQUATIONS_SCHEMA_V1} + row.update( + equation_id=eq_id, paper_id="2401.00001", + presentation_latex="x", mathml="x", + mathml_tree_json=tree, + ) + rows.append(row) + table.add(pa.Table.from_pylist(rows, schema=EQUATIONS_SCHEMA_V1)) + return path + + +def _trees(path: Path) -> dict[str, str]: + table = lancedb.connect(str(path)).open_table("equations").to_arrow() + return dict(zip( + table.column("equation_id").to_pylist(), + table.column("mathml_tree_json").to_pylist(), + strict=True, + )) + + +def test_equations_command_fills_missing_trees_and_keeps_others(tmp_path): + path = _equations_db(tmp_path) + + result = _run("ingest.index_equations", "--lancedb-path", str(path)) + + assert result.returncode == 0, result.stderr + trees = _trees(path) + assert json.loads(trees["eq-new"]) + assert trees["eq-stale"] == "stale tree" + + +def test_equations_rebuild_replaces_stale_trees(tmp_path): + path = _equations_db(tmp_path) + + result = _run("ingest.index_equations", "--lancedb-path", str(path), "--rebuild") + + assert result.returncode == 0, result.stderr + trees = _trees(path) + assert trees["eq-stale"] == trees["eq-new"] + assert json.loads(trees["eq-stale"]) + + +def test_definitions_command_is_runnable(): + result = _run("ingest.index_definitions", "--help") + + assert result.returncode == 0 + assert "usage: python -m ingest.index_definitions" in result.stdout + + +def test_definitions_command_indexes_every_paper_in_the_chunks_table( + tmp_path, capsys +): + lancedb_path = _chunks_db(tmp_path) + preamble_dir = tmp_path / "preamble" + for paper_id in ("2401.00001", "2401.00002"): + doc = PreambleDoc( + paper_id=paper_id, source_hash="0" * 64, + macros=[r"\newcommand{\RR}{\mathbb{R}}"], + preamble_text="", preamble_hash="0" * 16, + ) + (preamble_dir / paper_id).mkdir(parents=True) + (preamble_dir / paper_id / "preamble.json").write_text( + json.dumps(doc.to_dict()), encoding="utf-8" + ) + + with patch("ingest.preamble.PREAMBLE_DIR", preamble_dir): + assert index_definitions._cli(["--lancedb-path", str(lancedb_path)]) == 0 + + assert "indexed 2 definition(s) across 2 paper(s)" in capsys.readouterr().out + table = lancedb.connect(str(lancedb_path)).open_table("definitions").to_arrow() + assert sorted(table.column("paper_id").to_pylist()) == ["2401.00001", "2401.00002"]