Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 34 additions & 23 deletions ingest/index_theorem_names.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,25 +184,44 @@ async def index_theorem_names_all_papers(


def default_db_path(lancedb_path: str | Path) -> Path:
"""Return the canonical on-disk path for the theorem-names DB.
"""Where the theorem-name index of the dataset at ``lancedb_path`` lives:
inside the dataset, so cutover and rollback move it with the chunks it
indexes. The server derives the same path."""
from server.application_paths import derived_index_path

Sibling-of-sibling to the LanceDB dataset root:
``var/arxmcp/index/sqlite/theorem_names.db``. This keeps the
project's ``var/`` tree organized — LanceDB indices live under
``index/lancedb/``, SQLite indices under ``index/sqlite/``.
"""
return (
Path(lancedb_path).resolve().parent / "sqlite" / "theorem_names.db"
)
return derived_index_path(Path(lancedb_path).resolve(), "theorem_names.db")



def _cli(argv: list[str]) -> int:
import argparse
def build_theorem_names_index(
lancedb_path: str | Path, db_path: str | Path | None = None,
) -> dict[str, int]:
"""Rebuild the theorem-name index of the dataset at ``lancedb_path``.

Writes to ``db_path``, by default :func:`default_db_path`, migrating an
out-of-date file, which only an offline caller may do. Returns
``{paper_id: rows_written}``.
"""
import asyncio

import lancedb

target = Path(db_path) if db_path is not None else default_db_path(lancedb_path)
chunks = lancedb.connect(str(lancedb_path)).open_table("chunks")

async def run() -> dict[str, int]:
store = await TheoremNamesStore.open(target)
try:
return await index_theorem_names_all_papers(chunks, store)
finally:
await store.close()

return asyncio.run(run())


def _cli(argv: list[str]) -> int:
import argparse

from ingest.store import DEFAULT_LANCEDB_PATH

parser = argparse.ArgumentParser(
Expand All @@ -215,21 +234,12 @@ def _cli(argv: list[str]) -> int:
)
parser.add_argument(
"--db-path", default=None,
help="SQLite index to write (default: sqlite/theorem_names.db "
"beside the LanceDB root, where the server reads it)",
help="SQLite index to write (default: derived/theorem_names.db "
"inside the LanceDB dataset, where the server reads it)",
)
args = parser.parse_args(argv)
db_path = Path(args.db_path) if args.db_path else default_db_path(args.lancedb_path)
chunks = lancedb.connect(args.lancedb_path).open_table("chunks")

async def run() -> dict[str, int]:
store = await TheoremNamesStore.open(db_path)
try:
return await index_theorem_names_all_papers(chunks, store)
finally:
await store.close()

counts = asyncio.run(run())
counts = build_theorem_names_index(args.lancedb_path, db_path)
print(
f"indexed {sum(counts.values())} theorem name(s) across "
f"{len(counts)} paper(s) into {db_path}"
Expand All @@ -243,6 +253,7 @@ async def run() -> dict[str, int]:
sys.exit(_cli(sys.argv[1:]))

__all__ = [
"build_theorem_names_index",
"DEFAULT_CONFIDENCE",
"build_rows_from_chunks_for_paper",
"default_db_path",
Expand Down
9 changes: 7 additions & 2 deletions ops/checkpoint_notebooks_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,13 +119,18 @@ def checkpoint(
def backup_set_databases(var_root: Path) -> list[Path]:
"""Every SQLite database the backup copies, under ``var_root``.

That is ``cache/notebooks.db`` plus each notebook's stores under
``notebooks/`` (``documents.db``, ``paper_metadata.db``). Each runs in
That is ``cache/notebooks.db``, the shared dataset's derived indexes
under ``index/lancedb/derived/``, and each notebook's stores under
``notebooks/`` (``documents.db``, ``paper_metadata.db``, and its
dataset's ``lancedb/derived/theorem_names.db``). Each runs in
WAL mode, so each needs the same checkpoint. A notebook's
``cache/retrieval.db`` is a regenerable query cache that the backup
excludes, so it is skipped here too.
"""
databases = [var_root / "cache" / "notebooks.db"]
# The shared dataset's derived indexes (the theorem-name index) sit
# inside index/lancedb, which the backup copies.
databases += sorted((var_root / "index" / "lancedb" / "derived").glob("*.db"))
notebooks = var_root / "notebooks"
if notebooks.is_dir():
databases += sorted(
Expand Down
19 changes: 18 additions & 1 deletion server/application_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
("corpus", "corpus"), ("index", "index"),
("lancedb", "index/lancedb"), ("kuzu", "index/kuzu"),
("bm25", "index/bm25"), ("sqlite", "index/sqlite"),
("theorem_names_db", "index/sqlite/theorem_names.db"),
("theorem_names_db", "index/lancedb/derived/theorem_names.db"),
("notebooks", "notebooks"), ("cache", "cache"),
("retrieval_cache_db", "cache/retrieval.db"),
("notebooks_db", "cache/notebooks.db"), ("ops", "ops"),
Expand All @@ -38,6 +38,18 @@ class ApplicationPathError(ValueError):
"""A configured application path is invalid or escapes its root."""


#: Indexes derived from a LanceDB dataset live in this subdirectory of it,
#: so that renaming the dataset (cutover, rollback) moves them too, and
#: each notebook's dataset carries its own. LanceDB ignores entries in its
#: root that are not tables.
DERIVED_INDEX_DIR = "derived"


def derived_index_path(lancedb_path: str | Path, name: str) -> Path:
"""Where the derived index ``name`` of the dataset at ``lancedb_path`` lives."""
return Path(lancedb_path) / DERIVED_INDEX_DIR / name


def legacy_source_path(name: str) -> Path:
"""Return the historical checkout-relative spelling for a layout key."""
suffixes = dict(_LAYOUT)
Expand Down Expand Up @@ -175,6 +187,11 @@ def resolve(
values[field] = resolved
else:
values[field] = _inside(canonical_root, path, field)
if not supplied.get(alias_by_field["theorem_names_db"]):
# Follows the dataset, including a dataset moved by its own alias.
values["theorem_names_db"] = derived_index_path(
values["lancedb"], "theorem_names.db"
)
return cls(
selected,
canonical_root,
Expand Down
20 changes: 20 additions & 0 deletions server/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
from server.application_paths import (
COMPATIBILITY_ALIAS_FIELDS,
ApplicationPaths,
derived_index_path,
legacy_source_path,
)
from server.corpus_freshness import DEFAULT_FRESHNESS_INTERVAL_SECONDS
Expand Down Expand Up @@ -719,6 +720,25 @@ def derive_notebook_lancedb_path(self) -> Config:
)
return self

@model_validator(mode="after")
def derive_dataset_index_paths(self) -> Config:
"""Place the theorem-name index inside the dataset being served.

Runs after notebook routing has settled :attr:`lancedb_path`, so a
notebook-routed server reads its own notebook's index, and the index
moves with the dataset when cutover or rollback renames it. An
explicit ``ARXMCP_THEOREM_NAMES_DB_PATH`` still wins.
"""
if "theorem_names_db_path" in self._supplied_path_fields:
return self
self.theorem_names_db_path = derived_index_path(
self.lancedb_path, "theorem_names.db"
)
self._application_paths = replace(
self._application_paths, theorem_names_db=self.theorem_names_db_path
)
return self

@model_validator(mode="after")
def reject_non_loopback_bind(self) -> Config:
"""Closes the brief's "binding to ``0.0.0.0`` is rejected at
Expand Down
2 changes: 2 additions & 0 deletions tests/test_checkpoint_notebooks_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,9 @@ def test_checkpoints_every_backed_up_database(self, tmp_path, capsys):
var = tmp_path / "var" / "arxmcp"
backed_up = [
var / "cache" / "notebooks.db",
var / "index" / "lancedb" / "derived" / "theorem_names.db",
var / "notebooks" / "demo" / "documents.db",
var / "notebooks" / "demo" / "lancedb" / "derived" / "theorem_names.db",
var / "notebooks" / "demo" / "paper_metadata.db",
]
query_cache = var / "notebooks" / "demo" / "cache" / "retrieval.db"
Expand Down
2 changes: 1 addition & 1 deletion tests/test_index_clis.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def test_theorem_names_command_builds_the_index_the_server_reads(tmp_path):

assert result.returncode == 0, result.stderr
assert "across 2 paper(s)" in result.stdout
db = tmp_path / "index" / "sqlite" / "theorem_names.db"
db = lancedb_path / "derived" / "theorem_names.db"
conn = sqlite3.connect(str(db))
try:
names = {r[0] for r in conn.execute("SELECT display_name FROM theorem_names")}
Expand Down
16 changes: 16 additions & 0 deletions tests/test_notebook_cutover.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,22 @@ def test_rollback_is_lossless_roundtrip(nb_base) -> None:
assert (nb / "lancedb-staging" / "_sentinel.txt").read_text() == "v645"


def test_derived_indexes_travel_with_their_dataset(nb_base) -> None:
"""An index built inside a dataset is promoted and rolled back with it."""
nb = nb_base / "demo-nb"
for name, version in (("lancedb", 369), ("lancedb-staging", 645)):
_write_lancedb(nb / name, version=version)
(nb / name / "derived").mkdir()
(nb / name / "derived" / "theorem_names.db").write_text(f"v{version}")
index = nb / "lancedb" / "derived" / "theorem_names.db"

nc.perform_cutover("demo-nb", base=nb_base)
assert index.read_text() == "v645"

nc.perform_rollback("demo-nb", base=nb_base)
assert index.read_text() == "v369"


def test_rollback_without_backup_refused(nb_base) -> None:
"""FM-6: rollback with no lancedb-prev-* backup refuses, no mutation."""
nb = nb_base / "demo-nb"
Expand Down
21 changes: 21 additions & 0 deletions tests/test_notebook_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

import json
import sqlite3
from types import SimpleNamespace

import lancedb
Expand All @@ -22,6 +23,7 @@ def _chunk(paper_id: str) -> dict:
chunk_id=f"{paper_id}#c0", paper_id=paper_id, kind="theorem",
section_path=["1"], body_text="Let X be a scheme.",
body_tokens="let x be a scheme", chunker_version="v",
theorem_name=f"Lemma of {paper_id}",
embedder_version="v",
)
return row
Expand Down Expand Up @@ -71,3 +73,22 @@ def test_ingest_builds_the_definitions_table(tmp_path, monkeypatch):
)
assert sorted(definitions.column("paper_id").to_pylist()) == sorted(PAPERS)
assert set(definitions.column("symbol").to_pylist()) == {r"\RR"}


def test_ingest_builds_the_theorem_name_index(tmp_path, monkeypatch):
base = tmp_path / "notebooks"
nb_dir = base / "demo-nb"
nb_dir.mkdir(parents=True)
(nb_dir / "papers.txt").write_text("\n".join(PAPERS) + "\n", encoding="utf-8")
monkeypatch.setattr(_notebook_common, "NOTEBOOKS_BASE", base)
monkeypatch.setattr("ingest.preamble.PREAMBLE_DIR", tmp_path / "preamble")
monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_bulk_ingest)

assert notebook_ingest.run("demo-nb") == 0

conn = sqlite3.connect(str(nb_dir / "lancedb" / "derived" / "theorem_names.db"))
try:
names = {r[0] for r in conn.execute("SELECT display_name FROM theorem_names")}
finally:
conn.close()
assert names == {f"Lemma of {pid}" for pid in PAPERS}
28 changes: 28 additions & 0 deletions tests/test_server_startup.py
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,34 @@ def test_notebook_set_derives_lancedb_path(self, notebooks_base):
cfg = Config(notebook="demo-nb")
assert cfg.lancedb_path == (notebooks_base / "demo-nb" / "lancedb").resolve()

def test_shared_theorem_names_live_in_the_shared_dataset(self, monkeypatch):
monkeypatch.delenv("ARXMCP_NOTEBOOK", raising=False)
monkeypatch.delenv("ARXMCP_THEOREM_NAMES_DB_PATH", raising=False)
cfg = Config()
assert cfg.theorem_names_db_path == Path(
"var/arxmcp/index/lancedb/derived/theorem_names.db"
)

def test_notebook_reads_its_own_theorem_names(self, notebooks_base, monkeypatch):
monkeypatch.delenv("ARXMCP_THEOREM_NAMES_DB_PATH", raising=False)
_seed_notebook_marker(notebooks_base / "demo-nb" / "lancedb")
cfg = Config(notebook="demo-nb")
assert cfg.theorem_names_db_path == (
notebooks_base / "demo-nb" / "lancedb" / "derived" / "theorem_names.db"
).resolve()

def test_explicit_dataset_carries_its_theorem_names(self, tmp_path, monkeypatch):
monkeypatch.delenv("ARXMCP_THEOREM_NAMES_DB_PATH", raising=False)
cfg = Config(lancedb_path=tmp_path / "lancedb")
assert cfg.theorem_names_db_path == (
tmp_path / "lancedb" / "derived" / "theorem_names.db"
)

def test_explicit_theorem_names_path_wins(self, notebooks_base, tmp_path):
_seed_notebook_marker(notebooks_base / "demo-nb" / "lancedb")
cfg = Config(notebook="demo-nb", theorem_names_db_path=tmp_path / "t.db")
assert cfg.theorem_names_db_path == tmp_path / "t.db"

def test_notebook_set_via_env(self, notebooks_base, monkeypatch):
"""The slug arrives via ARXMCP_NOTEBOOK env, not just kwargs."""
_seed_notebook_marker(notebooks_base / "demo-nb" / "lancedb")
Expand Down
15 changes: 15 additions & 0 deletions tests/tools/test_notebook_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,9 @@ def _fake_ingest(paper_ids, **kw):
monkeypatch.setattr(
notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0)
)
monkeypatch.setattr(
notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {}
)

rc = notebook_ingest.run("demo")
assert rc == 0
Expand Down Expand Up @@ -689,6 +692,9 @@ def _capture_build(*args, **kwargs):
monkeypatch.setattr(
notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0)
)
monkeypatch.setattr(
notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {}
)

rc = notebook_ingest.run("myslug")
assert rc == 0
Expand Down Expand Up @@ -746,6 +752,9 @@ def _capture_root(*args, **kwargs):
monkeypatch.setattr(
notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0)
)
monkeypatch.setattr(
notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {}
)
monkeypatch.setattr(notebook_ingest, "run_bulk_ingest", _fake_ingest_alpha)
notebook_ingest.run("alpha")

Expand Down Expand Up @@ -801,6 +810,9 @@ def _fake_build(*args, **kwargs):
monkeypatch.setattr(
notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0)
)
monkeypatch.setattr(
notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {}
)
notebook_ingest.run("demo")
nb_bm25_root = actual_root_holder[0]
assert nb_bm25_root is not None
Expand All @@ -815,6 +827,9 @@ def _fake_build(*args, **kwargs):
monkeypatch.setattr(
notebook_ingest, "index_definitions_for_corpus", lambda *a, **kw: (0, 0)
)
monkeypatch.setattr(
notebook_ingest, "build_theorem_names_index", lambda *a, **kw: {}
)
rc = notebook_ingest.run("demo")
assert rc == 0
err = capsys.readouterr().err
Expand Down
6 changes: 6 additions & 0 deletions tools/notebook_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
from ingest.bm25_indexer import build_bm25_index
from ingest.bulk_ingest import run_bulk_ingest
from ingest.index_definitions import index_definitions_for_corpus
from ingest.index_theorem_names import build_theorem_names_index
from tools._notebook_common import (
NotebookError,
notebook_dir,
Expand Down Expand Up @@ -187,6 +188,11 @@ def run(slug: str) -> int:
# index_status="absent" for every freshly ingested notebook.
rows, papers = index_definitions_for_corpus(lancedb_path)
print(f"definitions indexed: {rows} across {papers} paper(s)")
# find_lemma_by_name reads the theorem-name index inside the same dataset.
names = build_theorem_names_index(lancedb_path)
print(
f"theorem names indexed: {sum(names.values())} across {len(names)} paper(s)"
)

# Warn if multiple v<N>/ directories exist under the per-notebook
# BM25 root. Helps operators prune stale indices after a
Expand Down
Loading