Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions ingest/bm25_indexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@

from rank_bm25 import BM25Okapi

from server.corpus import open_chunks_table
from server.corpus import open_chunks_table, project_columns

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -303,6 +303,9 @@ def build_bm25_index(
FileNotFoundError
``lancedb_path`` does not exist on disk (re-raised from
:func:`open_chunks_table`).
RuntimeError
The projected scan returned fewer rows than the pinned version
holds.
"""
# Closes L3 from the E04_S04 critique: defense-in-depth validation
# of the version integer. ``_bm25_version_dir(-1)`` would produce
Expand Down Expand Up @@ -352,7 +355,21 @@ def build_bm25_index(
# validates the path, performs ``tbl.checkout(version)`` for MVCC
# isolation, and re-raises bad-version errors as ``ValueError``.
tbl = open_chunks_table(lancedb_path, version=corpus_version)
arrow = tbl.to_arrow()
# Project at the scanner: `tbl.to_arrow()` would materialize both
# 1024-float vector columns for every row to read three string
# columns. The table is checked out at `corpus_version`, so a short
# read is a failed read, and an index built from it would silently
# omit chunks; refuse instead.
arrow = project_columns(
tbl,
["chunk_id", "body_tokens", "paper_id"],
expected_rows=tbl.count_rows(),
)
if arrow is None:
raise RuntimeError(
f"chunks scan came back short at corpus_version={corpus_version}; "
"refusing to build a BM25 index from a partial read"
)
raw_chunk_ids = arrow.column("chunk_id").to_pylist()
raw_body_tokens = arrow.column("body_tokens").to_pylist()
raw_paper_ids = arrow.column("paper_id").to_pylist()
Expand Down
28 changes: 13 additions & 15 deletions ops/restore_drill_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,18 +60,16 @@ def _locate_lancedb_root(restore_path: Path) -> Path | None:
def _locate_kuzu_root(restore_path: Path) -> Path | None:
"""Find the restored Kùzu graph by searching for the
canonical ``var/arxmcp/index/kuzu`` segment under
``restore_path``. Returns ``None`` if no Kùzu directory
``restore_path``. Returns ``None`` if no Kùzu graph
was captured.

Closes adversary F1 (Kùzu half).
"""
# Kùzu doesn't have a single canonical marker file, so we
# look for the directory name. Filter to directories whose
# immediate parent is the expected ``index/`` so we don't
# match arbitrary "kuzu" directories the operator may have
# added under the snapshot.
# Kùzu 0.11 stores the database as a single file at this path, so
# match the name whether it is a file or a directory. Filter to
# entries whose immediate parent is the expected ``index/`` so we
# don't match arbitrary "kuzu" paths the operator may have added
# under the snapshot.
for entry in restore_path.rglob("kuzu"):
if entry.is_dir() and entry.parent.name == "index":
if entry.parent.name == "index":
return entry
return None

Expand Down Expand Up @@ -118,17 +116,17 @@ def smoke_check_lancedb(restore_path: Path) -> int:
def smoke_check_kuzu(restore_path: Path) -> int | None:
"""Open the restored Kùzu graph and return paper count.

Returns ``None`` if the Kùzu directory is absent (acceptable
Returns ``None`` if the Kùzu graph is absent (acceptable
— some restore paths may exclude the citation graph). Raises
``RuntimeError`` if the directory exists but is unreadable.
``RuntimeError`` if the graph exists but is unreadable.

Closes adversary F1 (Kùzu half): finds the Kùzu directory
via rglob rather than hardcoding the path prefix.
Finds the graph via rglob rather than hardcoding the path
prefix, because restic restores under the absolute source path.
"""
kuzu_path = _locate_kuzu_root(restore_path)
if kuzu_path is None:
logger.info(
"restore_drill: Kùzu directory absent under %s — skipping",
"restore_drill: Kùzu graph absent under %s — skipping",
restore_path,
)
return None
Expand All @@ -145,7 +143,7 @@ def smoke_check_kuzu(restore_path: Path) -> int | None:
db = kuzu.Database(str(kuzu_path), read_only=True)
conn = kuzu.Connection(db)
# Restored DB may be empty; we only require it to open.
result = conn.execute("MATCH (p:Paper) RETURN count(p) AS c")
result = conn.execute("MATCH (p:papers) RETURN count(p) AS c")
paper_count = 0
while result.has_next():
row = result.get_next()
Expand Down
37 changes: 23 additions & 14 deletions server/handlers/chunk.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
from server.observability.sanitize import sanitize_retrieved_text
from server.proof_linkage import resolve_referenced
from server.tools import (
cap_result_list,
enforce_byte_cap,
envelope,
get_resources,
Expand Down Expand Up @@ -78,6 +79,9 @@ async def handle_get_chunk(
ids returns ``{chunks: [<the same per-chunk object>, ...]}`` in
request order — one entry per id, each carrying its own ``found``,
byte-cap, and linkage — so N ids cost one round-trip instead of N.
If the assembled batch is still over the byte cap, trailing entries
are dropped and ``body_truncated`` is set at the top level; the ids
past the last entry can be fetched again.

A malformed id raises a JSON-RPC error; a well-formed id absent from
the corpus yields a per-chunk ``found: false`` (batch elements stay
Expand Down Expand Up @@ -113,21 +117,26 @@ async def handle_get_chunk(
return envelope(
_build_chunk_result(ids[0], row_map.get(ids[0]), include_referenced, r)
)
# Per-element byte budget so the whole chunks[] array is bounded by
# the per-response cap, not n independent copies of it. Each element
# gets result_byte_cap // n; a chunk over its share is truncated to a
# resource_link exactly as a scalar over-cap chunk is, so every
# requested id is still returned.
# Per-element byte budget: each element gets result_byte_cap // n,
# and a primary body over its share is truncated to a resource_link
# exactly as a scalar over-cap chunk is. That share bounds only the
# primary body: referenced counterparts (up to MAX_REFERENCED bodies
# each) are never trimmed by it, so the assembled array is capped
# again as a whole, dropping trailing entries until it fits.
per_elem_cap = getattr(r.config, "result_byte_cap", 256 * 1024) // len(ids)
return envelope({
"chunks": [
_build_chunk_result(
cid, row_map.get(cid), include_referenced, r,
byte_cap=per_elem_cap,
)
for cid in ids
]
})
structured, _ = cap_result_list(
{
"chunks": [
_build_chunk_result(
cid, row_map.get(cid), include_referenced, r,
byte_cap=per_elem_cap,
)
for cid in ids
]
},
list_key="chunks",
)
return envelope(structured)


def _fetch_chunk_rows(r, ids: list[str]) -> dict[str, Any]: # noqa: ANN001
Expand Down
8 changes: 7 additions & 1 deletion server/handlers/paper.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@

logger = logging.getLogger(__name__)

#: The only chunk columns ``get_paper`` reads; the row count comes free.
_PAPER_CHUNK_COLUMNS = ["section_path", "chunker_version", "embedder_version"]


def hydrated_paper_fields(record: Any) -> dict[str, Any] | None:
"""Map a ``PaperMetadataRecord`` to the ``get_paper`` identity
Expand Down Expand Up @@ -164,10 +167,13 @@ async def handle_get_paper(
# Filter to rows matching paper_id. LanceDB doesn't expose a
# native group-by; pull the matching rows in Arrow and aggregate
# in Python. For the 50-paper corpus the worst case is ~100s of
# rows per paper — fast enough.
# rows per paper — fast enough. Project at the scanner: an
# unprojected read pulls both 1024-float vector columns (~14 KB/row)
# for every row only to count them and read two version strings.
arrow = (
r.chunks_table.search()
.where(f"paper_id = '{_escape(paper_id)}'", prefilter=True)
.select(_PAPER_CHUNK_COLUMNS)
.limit(10000)
.to_arrow()
)
Expand Down
Loading
Loading