diff --git a/ingest/index_theorem_names.py b/ingest/index_theorem_names.py index 4d21255..004100f 100644 --- a/ingest/index_theorem_names.py +++ b/ingest/index_theorem_names.py @@ -184,25 +184,44 @@ async def index_theorem_names_all_papers( def default_db_path(lancedb_path: str | Path) -> Path: - """Return the canonical on-disk path for the theorem-names DB. + """Where the theorem-name index of the dataset at ``lancedb_path`` lives: + inside the dataset, so cutover and rollback move it with the chunks it + indexes. The server derives the same path.""" + from server.application_paths import derived_index_path - Sibling-of-sibling to the LanceDB dataset root: - ``var/arxmcp/index/sqlite/theorem_names.db``. This keeps the - project's ``var/`` tree organized — LanceDB indices live under - ``index/lancedb/``, SQLite indices under ``index/sqlite/``. - """ - return ( - Path(lancedb_path).resolve().parent / "sqlite" / "theorem_names.db" - ) + return derived_index_path(Path(lancedb_path).resolve(), "theorem_names.db") -def _cli(argv: list[str]) -> int: - import argparse +def build_theorem_names_index( + lancedb_path: str | Path, db_path: str | Path | None = None, +) -> dict[str, int]: + """Rebuild the theorem-name index of the dataset at ``lancedb_path``. + + Writes to ``db_path``, by default :func:`default_db_path`, migrating an + out-of-date file, which only an offline caller may do. Returns + ``{paper_id: rows_written}``. + """ import asyncio import lancedb + target = Path(db_path) if db_path is not None else default_db_path(lancedb_path) + chunks = lancedb.connect(str(lancedb_path)).open_table("chunks") + + async def run() -> dict[str, int]: + store = await TheoremNamesStore.open(target) + try: + return await index_theorem_names_all_papers(chunks, store) + finally: + await store.close() + + return asyncio.run(run()) + + +def _cli(argv: list[str]) -> int: + import argparse + from ingest.store import DEFAULT_LANCEDB_PATH parser = argparse.ArgumentParser( @@ -215,21 +234,12 @@ def _cli(argv: list[str]) -> int: ) parser.add_argument( "--db-path", default=None, - help="SQLite index to write (default: sqlite/theorem_names.db " - "beside the LanceDB root, where the server reads it)", + help="SQLite index to write (default: derived/theorem_names.db " + "inside the LanceDB dataset, where the server reads it)", ) args = parser.parse_args(argv) db_path = Path(args.db_path) if args.db_path else default_db_path(args.lancedb_path) - chunks = lancedb.connect(args.lancedb_path).open_table("chunks") - - async def run() -> dict[str, int]: - store = await TheoremNamesStore.open(db_path) - try: - return await index_theorem_names_all_papers(chunks, store) - finally: - await store.close() - - counts = asyncio.run(run()) + counts = build_theorem_names_index(args.lancedb_path, db_path) print( f"indexed {sum(counts.values())} theorem name(s) across " f"{len(counts)} paper(s) into {db_path}" @@ -243,6 +253,7 @@ async def run() -> dict[str, int]: sys.exit(_cli(sys.argv[1:])) __all__ = [ + "build_theorem_names_index", "DEFAULT_CONFIDENCE", "build_rows_from_chunks_for_paper", "default_db_path", diff --git a/ops/checkpoint_notebooks_db.py b/ops/checkpoint_notebooks_db.py index be4b0e8..84f8b0a 100644 --- a/ops/checkpoint_notebooks_db.py +++ b/ops/checkpoint_notebooks_db.py @@ -119,13 +119,18 @@ def checkpoint( def backup_set_databases(var_root: Path) -> list[Path]: """Every SQLite database the backup copies, under ``var_root``. - That is ``cache/notebooks.db`` plus each notebook's stores under - ``notebooks/`` (``documents.db``, ``paper_metadata.db``). Each runs in + That is ``cache/notebooks.db``, the shared dataset's derived indexes + under ``index/lancedb/derived/``, and each notebook's stores under + ``notebooks/`` (``documents.db``, ``paper_metadata.db``, and its + dataset's ``lancedb/derived/theorem_names.db``). Each runs in WAL mode, so each needs the same checkpoint. A notebook's ``cache/retrieval.db`` is a regenerable query cache that the backup excludes, so it is skipped here too. """ databases = [var_root / "cache" / "notebooks.db"] + # The shared dataset's derived indexes (the theorem-name index) sit + # inside index/lancedb, which the backup copies. + databases += sorted((var_root / "index" / "lancedb" / "derived").glob("*.db")) notebooks = var_root / "notebooks" if notebooks.is_dir(): databases += sorted( diff --git a/server/application_paths.py b/server/application_paths.py index 50c44f8..b74bab3 100644 --- a/server/application_paths.py +++ b/server/application_paths.py @@ -25,7 +25,7 @@ ("corpus", "corpus"), ("index", "index"), ("lancedb", "index/lancedb"), ("kuzu", "index/kuzu"), ("bm25", "index/bm25"), ("sqlite", "index/sqlite"), - ("theorem_names_db", "index/sqlite/theorem_names.db"), + ("theorem_names_db", "index/lancedb/derived/theorem_names.db"), ("notebooks", "notebooks"), ("cache", "cache"), ("retrieval_cache_db", "cache/retrieval.db"), ("notebooks_db", "cache/notebooks.db"), ("ops", "ops"), @@ -38,6 +38,18 @@ class ApplicationPathError(ValueError): """A configured application path is invalid or escapes its root.""" +#: Indexes derived from a LanceDB dataset live in this subdirectory of it, +#: so that renaming the dataset (cutover, rollback) moves them too, and +#: each notebook's dataset carries its own. LanceDB ignores entries in its +#: root that are not tables. +DERIVED_INDEX_DIR = "derived" + + +def derived_index_path(lancedb_path: str | Path, name: str) -> Path: + """Where the derived index ``name`` of the dataset at ``lancedb_path`` lives.""" + return Path(lancedb_path) / DERIVED_INDEX_DIR / name + + def legacy_source_path(name: str) -> Path: """Return the historical checkout-relative spelling for a layout key.""" suffixes = dict(_LAYOUT) @@ -175,6 +187,11 @@ def resolve( values[field] = resolved else: values[field] = _inside(canonical_root, path, field) + if not supplied.get(alias_by_field["theorem_names_db"]): + # Follows the dataset, including a dataset moved by its own alias. + values["theorem_names_db"] = derived_index_path( + values["lancedb"], "theorem_names.db" + ) return cls( selected, canonical_root, diff --git a/server/config.py b/server/config.py index ee0752d..2b70054 100644 --- a/server/config.py +++ b/server/config.py @@ -42,6 +42,7 @@ from server.application_paths import ( COMPATIBILITY_ALIAS_FIELDS, ApplicationPaths, + derived_index_path, legacy_source_path, ) from server.corpus_freshness import DEFAULT_FRESHNESS_INTERVAL_SECONDS @@ -719,6 +720,25 @@ def derive_notebook_lancedb_path(self) -> Config: ) return self + @model_validator(mode="after") + def derive_dataset_index_paths(self) -> Config: + """Place the theorem-name index inside the dataset being served. + + Runs after notebook routing has settled :attr:`lancedb_path`, so a + notebook-routed server reads its own notebook's index, and the index + moves with the dataset when cutover or rollback renames it. An + explicit ``ARXMCP_THEOREM_NAMES_DB_PATH`` still wins. + """ + if "theorem_names_db_path" in self._supplied_path_fields: + return self + self.theorem_names_db_path = derived_index_path( + self.lancedb_path, "theorem_names.db" + ) + self._application_paths = replace( + self._application_paths, theorem_names_db=self.theorem_names_db_path + ) + return self + @model_validator(mode="after") def reject_non_loopback_bind(self) -> Config: """Closes the brief's "binding to ``0.0.0.0`` is rejected at diff --git a/tests/test_checkpoint_notebooks_db.py b/tests/test_checkpoint_notebooks_db.py index 2f273f2..2464433 100644 --- a/tests/test_checkpoint_notebooks_db.py +++ b/tests/test_checkpoint_notebooks_db.py @@ -162,7 +162,9 @@ def test_checkpoints_every_backed_up_database(self, tmp_path, capsys): var = tmp_path / "var" / "arxmcp" backed_up = [ var / "cache" / "notebooks.db", + var / "index" / "lancedb" / "derived" / "theorem_names.db", var / "notebooks" / "demo" / "documents.db", + var / "notebooks" / "demo" / "lancedb" / "derived" / "theorem_names.db", var / "notebooks" / "demo" / "paper_metadata.db", ] query_cache = var / "notebooks" / "demo" / "cache" / "retrieval.db" diff --git a/tests/test_index_clis.py b/tests/test_index_clis.py index bbbf607..b109327 100644 --- a/tests/test_index_clis.py +++ b/tests/test_index_clis.py @@ -64,7 +64,7 @@ def test_theorem_names_command_builds_the_index_the_server_reads(tmp_path): assert result.returncode == 0, result.stderr assert "across 2 paper(s)" in result.stdout - db = tmp_path / "index" / "sqlite" / "theorem_names.db" + db = lancedb_path / "derived" / "theorem_names.db" conn = sqlite3.connect(str(db)) try: names = {r[0] for r in conn.execute("SELECT display_name FROM theorem_names")} diff --git a/tests/test_notebook_cutover.py b/tests/test_notebook_cutover.py index 6fb61f8..eda295f 100644 --- a/tests/test_notebook_cutover.py +++ b/tests/test_notebook_cutover.py @@ -170,6 +170,22 @@ def test_rollback_is_lossless_roundtrip(nb_base) -> None: assert (nb / "lancedb-staging" / "_sentinel.txt").read_text() == "v645" +def test_derived_indexes_travel_with_their_dataset(nb_base) -> None: + """An index built inside a dataset is promoted and rolled back with it.""" + nb = nb_base / "demo-nb" + for name, version in (("lancedb", 369), ("lancedb-staging", 645)): + _write_lancedb(nb / name, version=version) + (nb / name / "derived").mkdir() + (nb / name / "derived" / "theorem_names.db").write_text(f"v{version}") + index = nb / "lancedb" / "derived" / "theorem_names.db" + + nc.perform_cutover("demo-nb", base=nb_base) + assert index.read_text() == "v645" + + nc.perform_rollback("demo-nb", base=nb_base) + assert index.read_text() == "v369" + + def test_rollback_without_backup_refused(nb_base) -> None: """FM-6: rollback with no lancedb-prev-* backup refuses, no mutation.""" nb = nb_base / "demo-nb" diff --git a/tests/test_notebook_ingest.py b/tests/test_notebook_ingest.py index 7997c02..6b594ab 100644 --- a/tests/test_notebook_ingest.py +++ b/tests/test_notebook_ingest.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import sqlite3 from types import SimpleNamespace import lancedb @@ -22,6 +23,7 @@ def _chunk(paper_id: str) -> dict: chunk_id=f"{paper_id}#c0", paper_id=paper_id, kind="theorem", section_path=["1"], body_text="Let X be a scheme.", body_tokens="let x be a scheme", chunker_version="v", + theorem_name=f"Lemma of {paper_id}", embedder_version="v", ) return row @@ -71,3 +73,22 @@ def test_ingest_builds_the_definitions_table(tmp_path, monkeypatch): ) assert sorted(definitions.column("paper_id").to_pylist()) == sorted(PAPERS) assert set(definitions.column("symbol").to_pylist()) == {r"\RR"} + + +def test_ingest_builds_the_theorem_name_index(tmp_path, monkeypatch): + base = tmp_path / "notebooks" + nb_dir = base / "demo-nb" + nb_dir.mkdir(parents=True) + (nb_dir / "papers.txt").write_text("\n".join(PAPERS) + "\n", encoding="utf-8") + monkeypatch.setattr(_notebook_common, "NOTEBOOKS_BASE", base) + monkeypatch.setattr("ingest.preamble.PREAMBLE_DIR", tmp_path / "preamble") + monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_bulk_ingest) + + assert notebook_ingest.run("demo-nb") == 0 + + conn = sqlite3.connect(str(nb_dir / "lancedb" / "derived" / "theorem_names.db")) + try: + names = {r[0] for r in conn.execute("SELECT display_name FROM theorem_names")} + finally: + conn.close() + assert names == {f"Lemma of {pid}" for pid in PAPERS} diff --git a/tests/test_server_startup.py b/tests/test_server_startup.py index 1b17da0..a06d9d6 100644 --- a/tests/test_server_startup.py +++ b/tests/test_server_startup.py @@ -476,6 +476,34 @@ def test_notebook_set_derives_lancedb_path(self, notebooks_base): cfg = Config(notebook="demo-nb") assert cfg.lancedb_path == (notebooks_base / "demo-nb" / "lancedb").resolve() + def test_shared_theorem_names_live_in_the_shared_dataset(self, monkeypatch): + monkeypatch.delenv("ARXMCP_NOTEBOOK", raising=False) + monkeypatch.delenv("ARXMCP_THEOREM_NAMES_DB_PATH", raising=False) + cfg = Config() + assert cfg.theorem_names_db_path == Path( + "var/arxmcp/index/lancedb/derived/theorem_names.db" + ) + + def test_notebook_reads_its_own_theorem_names(self, notebooks_base, monkeypatch): + monkeypatch.delenv("ARXMCP_THEOREM_NAMES_DB_PATH", raising=False) + _seed_notebook_marker(notebooks_base / "demo-nb" / "lancedb") + cfg = Config(notebook="demo-nb") + assert cfg.theorem_names_db_path == ( + notebooks_base / "demo-nb" / "lancedb" / "derived" / "theorem_names.db" + ).resolve() + + def test_explicit_dataset_carries_its_theorem_names(self, tmp_path, monkeypatch): + monkeypatch.delenv("ARXMCP_THEOREM_NAMES_DB_PATH", raising=False) + cfg = Config(lancedb_path=tmp_path / "lancedb") + assert cfg.theorem_names_db_path == ( + tmp_path / "lancedb" / "derived" / "theorem_names.db" + ) + + def test_explicit_theorem_names_path_wins(self, notebooks_base, tmp_path): + _seed_notebook_marker(notebooks_base / "demo-nb" / "lancedb") + cfg = Config(notebook="demo-nb", theorem_names_db_path=tmp_path / "t.db") + assert cfg.theorem_names_db_path == tmp_path / "t.db" + def test_notebook_set_via_env(self, notebooks_base, monkeypatch): """The slug arrives via ARXMCP_NOTEBOOK env, not just kwargs.""" _seed_notebook_marker(notebooks_base / "demo-nb" / "lancedb") diff --git a/tests/tools/test_notebook_scripts.py b/tests/tools/test_notebook_scripts.py index 87a8377..c17209e 100644 --- a/tests/tools/test_notebook_scripts.py +++ b/tests/tools/test_notebook_scripts.py @@ -409,6 +409,9 @@ def _fake_ingest(paper_ids, **kw): monkeypatch.setattr( notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) ) + monkeypatch.setattr( + notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {} + ) rc = notebook_ingest.run("demo") assert rc == 0 @@ -689,6 +692,9 @@ def _capture_build(*args, **kwargs): monkeypatch.setattr( notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) ) + monkeypatch.setattr( + notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {} + ) rc = notebook_ingest.run("myslug") assert rc == 0 @@ -746,6 +752,9 @@ def _capture_root(*args, **kwargs): monkeypatch.setattr( notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) ) + monkeypatch.setattr( + notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {} + ) monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest_alpha) notebook_ingest.run("alpha") @@ -801,6 +810,9 @@ def _fake_build(*args, **kwargs): monkeypatch.setattr( notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) ) + monkeypatch.setattr( + notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {} + ) notebook_ingest.run("demo") nb_bm25_root = actual_root_holder[0] assert nb_bm25_root is not None @@ -815,6 +827,9 @@ def _fake_build(*args, **kwargs): monkeypatch.setattr( notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0) ) + monkeypatch.setattr( + notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {} + ) rc = notebook_ingest.run("demo") assert rc == 0 err = capsys.readouterr().err diff --git a/tools/notebook_ingest.py b/tools/notebook_ingest.py index 98a4de3..d57ed25 100644 --- a/tools/notebook_ingest.py +++ b/tools/notebook_ingest.py @@ -44,6 +44,7 @@ from ingest.bm25_indexer import build_bm25_index from ingest.bulk_ingest import run_bulk_ingest from ingest.index_definitions import index_definitions_for_corpus +from ingest.index_theorem_names import build_theorem_names_index from tools._notebook_common import ( NotebookError, notebook_dir, @@ -187,6 +188,11 @@ def run(slug: str) -> int: # index_status="absent" for every freshly ingested notebook. rows, papers = index_definitions_for_corpus(lancedb_path) print(f"definitions indexed: {rows} across {papers} paper(s)") + # find_lemma_by_name reads the theorem-name index inside the same dataset. + names = build_theorem_names_index(lancedb_path) + print( + f"theorem names indexed: {sum(names.values())} across {len(names)} paper(s)" + ) # Warn if multiple v/ directories exist under the per-notebook # BM25 root. Helps operators prune stale indices after a