Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ingest/bulk_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
try_cache,
)
from ingest.chunker import STRUCTURE_SIGNAL_CLASSES, chunk_paper
from ingest.derived_indexes import build_derived_indexes
from ingest.embedder import embed_paper
from ingest.identifiers import is_valid_arxiv_paper_id
from ingest.store import (
Expand Down Expand Up @@ -441,6 +442,10 @@ def run_bulk_ingest(
if n % progress_interval == 0 or n == len(work):
_log_progress(log_path, summary, paper_id)

# The dataset is not ready to promote until it carries its indexes.
if summary.papers_succeeded:
build_derived_indexes(lancedb_staging_path)

summary.elapsed_seconds = time.monotonic() - started

# corpus-integrity-observability-e3: write the ingest-summary.json
Expand Down
82 changes: 82 additions & 0 deletions ingest/derived_indexes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
"""Build every index derived from a LanceDB dataset, inside that dataset.

A dataset that carries its own indexes can be promoted, rolled back or
backed up as one directory. The producers of a dataset (bulk ingest,
re-embed) call :func:`build_derived_indexes` before they report the
dataset ready, so a cutover never promotes chunks without the indexes
that answer queries over them.

The equations table is not built here: it comes from equation
extraction, which runs per paper during parsing.
"""

from __future__ import annotations

import logging
import sys
from dataclasses import dataclass
from pathlib import Path

logger = logging.getLogger(__name__)


@dataclass(frozen=True, slots=True)
class DerivedIndexSummary:
corpus_version: int
definitions: int
definition_papers: int
theorem_names: int
theorem_name_papers: int


def build_derived_indexes(lancedb_path: str | Path) -> DerivedIndexSummary:
"""Build the BM25, definitions and theorem-name indexes of a dataset.

Each is built for the version in the dataset's ``corpus-version.json``
and written inside the dataset. Rebuilding is safe: BM25 skips a
version it already holds, and the other two replace their rows.
"""
from ingest.bm25_indexer import build_bm25_index
from ingest.index_definitions import index_definitions_for_corpus
from ingest.index_theorem_names import build_theorem_names_index
from server.corpus import read_corpus_version

path = Path(lancedb_path)
info = read_corpus_version(lancedb_path=path)
if info is None:
raise FileNotFoundError(f"no corpus-version.json in {path}")
build_bm25_index(path, corpus_version=info.version)
definitions, definition_papers = index_definitions_for_corpus(path)
names = build_theorem_names_index(path)
summary = DerivedIndexSummary(
corpus_version=info.version,
definitions=definitions,
definition_papers=definition_papers,
theorem_names=sum(names.values()),
theorem_name_papers=len(names),
)
logger.info("derived indexes built for %s: %s", path, summary)
return summary


def _cli(argv: list[str]) -> int:
import argparse

parser = argparse.ArgumentParser(
description="Rebuild the indexes stored inside a LanceDB dataset."
)
parser.add_argument("lancedb_path", type=Path)
args = parser.parse_args(argv)
s = build_derived_indexes(args.lancedb_path)
print(
f"corpus_version={s.corpus_version} "
f"definitions={s.definitions} ({s.definition_papers} papers) "
f"theorem_names={s.theorem_names} ({s.theorem_name_papers} papers)"
)
return 0


if __name__ == "__main__": # pragma: no cover
sys.exit(_cli(sys.argv[1:]))

__all__ = ["DerivedIndexSummary", "build_derived_indexes"]
8 changes: 8 additions & 0 deletions ingest/oai_delta.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,9 @@
_log_parser_failure,
ingest_one_paper,
)
from ingest.derived_indexes import build_derived_indexes
from ingest.identifiers import is_valid_arxiv_paper_id
from ingest.store import CORPUS_VERSION_MARKER_NAME

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -816,6 +818,12 @@ def run_delta(
else:
summary.records_failed += 1

# The staged dataset changed: rebuild its indexes before the run counts
# as done, so a cutover never promotes chunks without them.
changed = summary.records_ingested or summary.records_deleted
if changed and (lancedb_staging_path / CORPUS_VERSION_MARKER_NAME).is_file():
build_derived_indexes(lancedb_staging_path)

summary.elapsed_seconds = time.monotonic() - started
summary.budget_breached = summary.elapsed_seconds > budget_seconds
if summary.budget_breached:
Expand Down
6 changes: 6 additions & 0 deletions ingest/re_embed.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
from ingest.bulk_ingest import DEFAULT_LANCEDB_STAGING_PATH
from ingest.chunker import chunk_paper
from ingest.chunker_types import CHUNKER_VERSION, ChunkRecord
from ingest.derived_indexes import build_derived_indexes
from ingest.embedder import EMBEDDER_VERSION, embed_paper
from ingest.identifiers import is_valid_arxiv_paper_id, paper_id_from_chunk_id
from ingest.schema import (
Expand Down Expand Up @@ -792,6 +793,11 @@ def run_re_embed(
},
)

# Before the terminal sentinel, so a staging dataset marked complete
# always carries the indexes built from its own chunks.
if not dry_run and (staging_lancedb_path / CORPUS_VERSION_MARKER_NAME).is_file():
build_derived_indexes(staging_lancedb_path)

summary.elapsed_seconds = time.monotonic() - started

# Finalize state on success (no failures past the per-paper isolation).
Expand Down
38 changes: 38 additions & 0 deletions tests/test_bulk_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,17 @@
run_bulk_ingest,
)


@pytest.fixture(autouse=True)
def derived_builds(monkeypatch) -> list:
"""Record index builds: the outcomes here are mocks with no dataset."""
calls: list = []
monkeypatch.setattr(
"ingest.bulk_ingest.build_derived_indexes", lambda path: calls.append(path)
)
return calls


# ---------------------------------------------------------------------------
# _read_paper_ids
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -165,6 +176,33 @@ def test_all_papers_succeed(self, tmp_path):
assert summary.ar5iv_hit_rate == 1.0
assert not failures.exists() # no failures, no JSONL line

def test_builds_the_indexes_into_the_dataset_it_wrote(
self, tmp_path, derived_builds
):
staging = tmp_path / "staging"
with patch(
"ingest.bulk_ingest.ingest_one_paper",
side_effect=[_fail_outcome("2401.00001"), _ok_outcome("2401.00002")],
):
run_bulk_ingest(
["2401.00001", "2401.00002"], lancedb_staging_path=staging,
failures_path=tmp_path / "f.jsonl", log_path=tmp_path / "log",
ops_dir=tmp_path,
)
assert derived_builds == [staging]

def test_builds_nothing_when_no_paper_landed(self, tmp_path, derived_builds):
with patch(
"ingest.bulk_ingest.ingest_one_paper",
side_effect=[_fail_outcome("2401.00001")],
):
run_bulk_ingest(
["2401.00001"], lancedb_staging_path=tmp_path / "staging",
failures_path=tmp_path / "f.jsonl", log_path=tmp_path / "log",
ops_dir=tmp_path,
)
assert derived_builds == []

def test_writes_ingest_summary_sentinel(self, tmp_path):
"""corpus-integrity-observability-e3 F3: run_bulk_ingest writes the
ingest-summary.json sentinel at end-of-run (AC2). The driver WIRING was
Expand Down
116 changes: 116 additions & 0 deletions tests/test_derived_indexes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
"""``ingest.derived_indexes`` builds a dataset's indexes inside it."""

from __future__ import annotations

import json
import sqlite3
from pathlib import Path

import lancedb
import pyarrow as pa

from ingest.derived_indexes import build_derived_indexes
from ingest.preamble import PreambleDoc
from ingest.schema import CHUNKS_SCHEMA_V1
from ingest.store import write_corpus_version_marker

PAPERS = ("2401.00001", "2401.00002")


def _chunk(paper_id: str) -> dict:
row = {field.name: None for field in CHUNKS_SCHEMA_V1}
row.update(
chunk_id=f"{paper_id}#c0", paper_id=paper_id, kind="theorem",
section_path=["1"], body_text="Let X be a scheme.",
body_tokens="let x be a scheme", chunker_version="v",
theorem_name=f"Lemma of {paper_id}", embedder_version="v",
)
return row


def _dataset(tmp_path, monkeypatch, name="lancedb-staging", papers=PAPERS):
"""A dataset (chunks + marker) and the preambles of its papers."""
staging = tmp_path / name
rows = [_chunk(pid) for pid in papers]
table = lancedb.connect(str(staging)).create_table(
"chunks", data=pa.Table.from_pylist(rows, schema=CHUNKS_SCHEMA_V1)
)
write_corpus_version_marker(
staging, version=table.version, chunker_version="v", embedder_version="v",
paper_count=len(rows), chunk_count=len(rows),
)
preamble_dir = tmp_path / "preamble"
for paper_id in papers:
doc = PreambleDoc(
paper_id=paper_id, source_hash="0" * 64,
macros=[r"\newcommand{\RR}{\mathbb{R}}"],
preamble_text="", preamble_hash="0" * 16,
)
(preamble_dir / paper_id).mkdir(parents=True, exist_ok=True)
(preamble_dir / paper_id / "preamble.json").write_text(
json.dumps(doc.to_dict()), encoding="utf-8"
)
monkeypatch.setattr("ingest.preamble.PREAMBLE_DIR", preamble_dir)
return staging


def test_a_staged_dataset_carries_its_own_indexes(tmp_path, monkeypatch):
staging = _dataset(tmp_path, monkeypatch)

summary = build_derived_indexes(staging)

# Nothing lands beside the dataset but the build-stats log.
assert {p.name for p in tmp_path.iterdir()} == {"lancedb-staging", "preamble", "ops"}
bm25 = staging / "derived" / "bm25" / f"v{summary.corpus_version}"
assert sorted(json.loads((bm25 / "chunk_ids.json").read_text())) == [
f"{pid}#c0" for pid in PAPERS
]
definitions = (
lancedb.connect(str(staging)).open_table("definitions").to_arrow()
)
assert sorted(definitions.column("paper_id").to_pylist()) == sorted(PAPERS)
assert set(definitions.column("symbol").to_pylist()) == {r"\RR"}
conn = sqlite3.connect(str(staging / "derived" / "theorem_names.db"))
try:
names = {r[0] for r in conn.execute("SELECT display_name FROM theorem_names")}
finally:
conn.close()
assert names == {f"Lemma of {pid}" for pid in PAPERS}


def test_rebuilding_is_safe(tmp_path, monkeypatch):
staging = _dataset(tmp_path, monkeypatch)
first = build_derived_indexes(staging)
assert build_derived_indexes(staging) == first


def _snapshot(dataset: Path) -> dict[str, bytes]:
derived = dataset / "derived"
return {
str(f.relative_to(derived)): f.read_bytes()
for f in sorted(derived.rglob("*")) if f.is_file()
}


def test_cutover_then_rollback_restores_the_live_indexes(tmp_path, monkeypatch):
from ops.cutover import perform_directory_swap, perform_rollback
from server.config import Config

live = _dataset(tmp_path, monkeypatch, "lancedb", PAPERS[:1])
build_derived_indexes(live)
staging = _dataset(tmp_path, monkeypatch, "lancedb-staging", PAPERS)
build_derived_indexes(staging)
before = _snapshot(live)
staged = _snapshot(staging)
served = Config(lancedb_path=live)

perform_directory_swap(
active_path=live, staging_path=staging,
rollback_path=tmp_path / "lancedb-prev",
)
assert _snapshot(live) == staged
perform_rollback(active_path=live, rollback_path=tmp_path / "lancedb-prev")

assert _snapshot(live) == before
assert served.theorem_names_db_path.read_bytes() == before["theorem_names.db"]
assert served.bm25_index_root.is_relative_to(live)
Loading
Loading