diff --git a/ingest/bm25_indexer.py b/ingest/bm25_indexer.py index 7560d89..9f88189 100644 --- a/ingest/bm25_indexer.py +++ b/ingest/bm25_indexer.py @@ -84,7 +84,7 @@ from rank_bm25 import BM25Okapi -from server.corpus import open_chunks_table +from server.corpus import open_chunks_table, project_columns logger = logging.getLogger(__name__) @@ -303,6 +303,9 @@ def build_bm25_index( FileNotFoundError ``lancedb_path`` does not exist on disk (re-raised from :func:`open_chunks_table`). + RuntimeError + The projected scan returned fewer rows than the pinned version + holds. """ # Closes L3 from the E04_S04 critique: defense-in-depth validation # of the version integer. ``_bm25_version_dir(-1)`` would produce @@ -352,7 +355,21 @@ def build_bm25_index( # validates the path, performs ``tbl.checkout(version)`` for MVCC # isolation, and re-raises bad-version errors as ``ValueError``. tbl = open_chunks_table(lancedb_path, version=corpus_version) - arrow = tbl.to_arrow() + # Project at the scanner: `tbl.to_arrow()` would materialize both + # 1024-float vector columns for every row to read three string + # columns. The table is checked out at `corpus_version`, so a short + # read is a failed read, and an index built from it would silently + # omit chunks; refuse instead. + arrow = project_columns( + tbl, + ["chunk_id", "body_tokens", "paper_id"], + expected_rows=tbl.count_rows(), + ) + if arrow is None: + raise RuntimeError( + f"chunks scan came back short at corpus_version={corpus_version}; " + "refusing to build a BM25 index from a partial read" + ) raw_chunk_ids = arrow.column("chunk_id").to_pylist() raw_body_tokens = arrow.column("body_tokens").to_pylist() raw_paper_ids = arrow.column("paper_id").to_pylist() diff --git a/ops/restore_drill_check.py b/ops/restore_drill_check.py index 327fd3a..ea15d9d 100644 --- a/ops/restore_drill_check.py +++ b/ops/restore_drill_check.py @@ -60,18 +60,16 @@ def _locate_lancedb_root(restore_path: Path) -> Path | None: def _locate_kuzu_root(restore_path: Path) -> Path | None: """Find the restored Kùzu graph by searching for the canonical ``var/arxmcp/index/kuzu`` segment under - ``restore_path``. Returns ``None`` if no Kùzu directory + ``restore_path``. Returns ``None`` if no Kùzu graph was captured. - - Closes adversary F1 (Kùzu half). """ - # Kùzu doesn't have a single canonical marker file, so we - # look for the directory name. Filter to directories whose - # immediate parent is the expected ``index/`` so we don't - # match arbitrary "kuzu" directories the operator may have - # added under the snapshot. + # Kùzu 0.11 stores the database as a single file at this path, so + # match the name whether it is a file or a directory. Filter to + # entries whose immediate parent is the expected ``index/`` so we + # don't match arbitrary "kuzu" paths the operator may have added + # under the snapshot. for entry in restore_path.rglob("kuzu"): - if entry.is_dir() and entry.parent.name == "index": + if entry.parent.name == "index": return entry return None @@ -118,17 +116,17 @@ def smoke_check_lancedb(restore_path: Path) -> int: def smoke_check_kuzu(restore_path: Path) -> int | None: """Open the restored Kùzu graph and return paper count. - Returns ``None`` if the Kùzu directory is absent (acceptable + Returns ``None`` if the Kùzu graph is absent (acceptable — some restore paths may exclude the citation graph). Raises - ``RuntimeError`` if the directory exists but is unreadable. + ``RuntimeError`` if the graph exists but is unreadable. - Closes adversary F1 (Kùzu half): finds the Kùzu directory - via rglob rather than hardcoding the path prefix. + Finds the graph via rglob rather than hardcoding the path + prefix, because restic restores under the absolute source path. """ kuzu_path = _locate_kuzu_root(restore_path) if kuzu_path is None: logger.info( - "restore_drill: Kùzu directory absent under %s — skipping", + "restore_drill: Kùzu graph absent under %s — skipping", restore_path, ) return None @@ -145,7 +143,7 @@ def smoke_check_kuzu(restore_path: Path) -> int | None: db = kuzu.Database(str(kuzu_path), read_only=True) conn = kuzu.Connection(db) # Restored DB may be empty; we only require it to open. - result = conn.execute("MATCH (p:Paper) RETURN count(p) AS c") + result = conn.execute("MATCH (p:papers) RETURN count(p) AS c") paper_count = 0 while result.has_next(): row = result.get_next() diff --git a/server/handlers/chunk.py b/server/handlers/chunk.py index 360a5da..dbd545b 100644 --- a/server/handlers/chunk.py +++ b/server/handlers/chunk.py @@ -36,6 +36,7 @@ from server.observability.sanitize import sanitize_retrieved_text from server.proof_linkage import resolve_referenced from server.tools import ( + cap_result_list, enforce_byte_cap, envelope, get_resources, @@ -78,6 +79,9 @@ async def handle_get_chunk( ids returns ``{chunks: [, ...]}`` in request order — one entry per id, each carrying its own ``found``, byte-cap, and linkage — so N ids cost one round-trip instead of N. + If the assembled batch is still over the byte cap, trailing entries + are dropped and ``body_truncated`` is set at the top level; the ids + past the last entry can be fetched again. A malformed id raises a JSON-RPC error; a well-formed id absent from the corpus yields a per-chunk ``found: false`` (batch elements stay @@ -113,21 +117,26 @@ async def handle_get_chunk( return envelope( _build_chunk_result(ids[0], row_map.get(ids[0]), include_referenced, r) ) - # Per-element byte budget so the whole chunks[] array is bounded by - # the per-response cap, not n independent copies of it. Each element - # gets result_byte_cap // n; a chunk over its share is truncated to a - # resource_link exactly as a scalar over-cap chunk is, so every - # requested id is still returned. + # Per-element byte budget: each element gets result_byte_cap // n, + # and a primary body over its share is truncated to a resource_link + # exactly as a scalar over-cap chunk is. That share bounds only the + # primary body: referenced counterparts (up to MAX_REFERENCED bodies + # each) are never trimmed by it, so the assembled array is capped + # again as a whole, dropping trailing entries until it fits. per_elem_cap = getattr(r.config, "result_byte_cap", 256 * 1024) // len(ids) - return envelope({ - "chunks": [ - _build_chunk_result( - cid, row_map.get(cid), include_referenced, r, - byte_cap=per_elem_cap, - ) - for cid in ids - ] - }) + structured, _ = cap_result_list( + { + "chunks": [ + _build_chunk_result( + cid, row_map.get(cid), include_referenced, r, + byte_cap=per_elem_cap, + ) + for cid in ids + ] + }, + list_key="chunks", + ) + return envelope(structured) def _fetch_chunk_rows(r, ids: list[str]) -> dict[str, Any]: # noqa: ANN001 diff --git a/server/handlers/paper.py b/server/handlers/paper.py index 72f3086..4d930a7 100644 --- a/server/handlers/paper.py +++ b/server/handlers/paper.py @@ -57,6 +57,9 @@ logger = logging.getLogger(__name__) +#: The only chunk columns ``get_paper`` reads; the row count comes free. +_PAPER_CHUNK_COLUMNS = ["section_path", "chunker_version", "embedder_version"] + def hydrated_paper_fields(record: Any) -> dict[str, Any] | None: """Map a ``PaperMetadataRecord`` to the ``get_paper`` identity @@ -164,10 +167,13 @@ async def handle_get_paper( # Filter to rows matching paper_id. LanceDB doesn't expose a # native group-by; pull the matching rows in Arrow and aggregate # in Python. For the 50-paper corpus the worst case is ~100s of - # rows per paper — fast enough. + # rows per paper — fast enough. Project at the scanner: an + # unprojected read pulls both 1024-float vector columns (~14 KB/row) + # for every row only to count them and read two version strings. arrow = ( r.chunks_table.search() .where(f"paper_id = '{_escape(paper_id)}'", prefilter=True) + .select(_PAPER_CHUNK_COLUMNS) .limit(10000) .to_arrow() ) diff --git a/server/notebooks_store.py b/server/notebooks_store.py index ed970e2..367bedc 100644 --- a/server/notebooks_store.py +++ b/server/notebooks_store.py @@ -89,6 +89,32 @@ SCHEMA_VERSION: int = 6 +def _migrate(conn: sqlite3.Connection, version: int, *statements: str) -> None: + """Run one migration step and stamp ``user_version`` atomically. + + The connection is ``isolation_level=None`` (autocommit), so without an + explicit BEGIN every statement commits on its own. A crash or error + between two of a step's statements would leave the DB half-migrated + (``user_version`` still the old value, some columns already present), + so the next open re-runs the step and its first ALTER raises + ``sqlite3.OperationalError: duplicate column name``, crash-looping the + daemon at startup. SQLite DDL and the ``user_version`` write are + transactional, so BEGIN/COMMIT makes the whole step all-or-nothing. + """ + conn.execute("BEGIN") + try: + for statement in statements: + conn.execute(statement) + conn.execute(f"PRAGMA user_version = {int(version)}") + conn.execute("COMMIT") + except BaseException: + # SQLite rolls some errors back on its own (SQLITE_FULL, for one); + # a second ROLLBACK would raise and hide the original error. + if conn.in_transaction: + conn.execute("ROLLBACK") + raise + + #: Module-level so a SYNC CLI can reach the migration ladder. #: #: It used to be a closure inside ``NotebooksStore.open``, which meant the @@ -154,17 +180,17 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # EXISTS) plus the preceding DROP IF EXISTS preserves # the original safety on empty DBs. if current_version < 1: - conn.execute("DROP TABLE IF EXISTS notebook_papers") - conn.execute("DROP TABLE IF EXISTS notebooks") - conn.execute( + _migrate( + conn, + 1, + "DROP TABLE IF EXISTS notebook_papers", + "DROP TABLE IF EXISTS notebooks", "CREATE TABLE notebooks (" " slug TEXT PRIMARY KEY," " display_name TEXT NOT NULL DEFAULT ''," " lancedb_path TEXT NOT NULL," " created_at TEXT NOT NULL" - ")" - ) - conn.execute( + ")", "CREATE TABLE notebook_papers (" " slug TEXT NOT NULL," " paper_id TEXT NOT NULL," @@ -172,13 +198,10 @@ def open_sync(db_path: Path) -> sqlite3.Connection: " PRIMARY KEY (slug, paper_id)," " FOREIGN KEY (slug) REFERENCES notebooks(slug) " " ON DELETE CASCADE" - ")" - ) - conn.execute( + ")", "CREATE INDEX idx_notebook_papers_slug " - "ON notebook_papers(slug)" + "ON notebook_papers(slug)", ) - conn.execute("PRAGMA user_version = 1") # v1 -> v2: m9 ADDITIVE migration. notebook_ingest_runs # stores per-ingest-trigger run state. CREATE TABLE IF # NOT EXISTS (NOT a destructive recreate) preserves all @@ -187,7 +210,9 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # pattern is acceptable for caches (miss, not data # loss) but NOT for notebook metadata. if current_version < 2: - conn.execute( + _migrate( + conn, + 2, "CREATE TABLE IF NOT EXISTS notebook_ingest_runs (" " id INTEGER PRIMARY KEY AUTOINCREMENT," " slug TEXT NOT NULL," @@ -198,13 +223,10 @@ def open_sync(db_path: Path) -> sqlite3.Connection: " stderr_tail TEXT," " FOREIGN KEY (slug) REFERENCES notebooks(slug) " " ON DELETE CASCADE" - ")" - ) - conn.execute( + ")", "CREATE INDEX IF NOT EXISTS idx_runs_slug " - "ON notebook_ingest_runs(slug, id DESC)" + "ON notebook_ingest_runs(slug, id DESC)", ) - conn.execute("PRAGMA user_version = 2") # v2 -> v3: textbook-ingest-m3 ADDITIVE migration. # ``notebook_kind`` distinguishes arXiv-corpus notebooks # (the historical default) from textbook-corpus notebooks @@ -217,11 +239,12 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # ``NotebookCreate.notebook_kind`` field enforces the # ``{arxiv, textbook}`` enum domain at the write path. if current_version < 3: - conn.execute( + _migrate( + conn, + 3, "ALTER TABLE notebooks ADD COLUMN notebook_kind " - "TEXT NOT NULL DEFAULT 'arxiv'" + "TEXT NOT NULL DEFAULT 'arxiv'", ) - conn.execute("PRAGMA user_version = 3") # v3 -> v4: textbook-ingest-m6 ADDITIVE migration. Three # parse-tracking columns on the ``notebooks`` table: # @@ -238,19 +261,16 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # rendered ``index.html``. Empty until the parse task # reports complete. if current_version < 4: - conn.execute( + _migrate( + conn, + 4, "ALTER TABLE notebooks ADD COLUMN parse_status " - "TEXT NOT NULL DEFAULT 'skipped'" - ) - conn.execute( + "TEXT NOT NULL DEFAULT 'skipped'", "ALTER TABLE notebooks ADD COLUMN parse_error " - "TEXT NOT NULL DEFAULT ''" - ) - conn.execute( + "TEXT NOT NULL DEFAULT ''", "ALTER TABLE notebooks ADD COLUMN parsed_html_path " - "TEXT NOT NULL DEFAULT ''" + "TEXT NOT NULL DEFAULT ''", ) - conn.execute("PRAGMA user_version = 4") # v4 -> v5: notebook-paper-discovery-m1 ADDITIVE migration. # Two columns on ``notebooks`` give a notebook a machine- # readable research interest for topic-driven paper discovery @@ -259,34 +279,15 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # {math.AG, math.NT, math-ph, hep-th}) and free-text # ``description``. Column-level DEFAULT '' backfills every # existing row in O(1) (no row-rewrite). - # - # Unlike the v1->v4 blocks (each a single ALTER), v4->v5 adds - # TWO columns. A crash BETWEEN the two ALTERs would leave the - # DB half-migrated (``user_version`` still 4 but - # ``discovery_category`` already present), so the next open - # re-runs the block and the first ALTER raises - # ``sqlite3.OperationalError: duplicate column name``, - # crash-looping the daemon at startup. Wrap the two ALTERs + - # the version bump in an explicit BEGIN/COMMIT so the whole - # step is atomic and re-runnable. The connection is - # ``isolation_level=None`` (autocommit), so the transaction - # MUST be opened explicitly. if current_version < 5: - conn.execute("BEGIN") - try: - conn.execute( - "ALTER TABLE notebooks ADD COLUMN " - "discovery_category TEXT NOT NULL DEFAULT ''" - ) - conn.execute( - "ALTER TABLE notebooks ADD COLUMN " - "description TEXT NOT NULL DEFAULT ''" - ) - conn.execute("PRAGMA user_version = 5") - conn.execute("COMMIT") - except Exception: - conn.execute("ROLLBACK") - raise + _migrate( + conn, + 5, + "ALTER TABLE notebooks ADD COLUMN " + "discovery_category TEXT NOT NULL DEFAULT ''", + "ALTER TABLE notebooks ADD COLUMN " + "description TEXT NOT NULL DEFAULT ''", + ) # v5 -> v6: contract-v1 ADDITIVE migration. # ``formal_releases`` is arXMCP's PIN to a topic repo's # released formalization. ADR-0004: arXMCP hosts no @@ -308,37 +309,32 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # and 4.9 rule 1 forbids collapsing that into a pass/fail # token. See ``tools/formal_release_pin.py``. if current_version < 6: - conn.execute("BEGIN") - try: - conn.execute( - "CREATE TABLE IF NOT EXISTS formal_releases (" - " slug TEXT NOT NULL," - " repo TEXT NOT NULL," - " tag TEXT NOT NULL," - " tag_object_sha TEXT NOT NULL," - " commit_sha TEXT NOT NULL," - " registry_id TEXT NOT NULL," - " registry_sha256 TEXT NOT NULL," - " env_digest TEXT NOT NULL," - " digest_provenance TEXT NOT NULL," - " asset_dir TEXT NOT NULL DEFAULT ''," - " bundle_json TEXT NOT NULL," - " registry_json TEXT NOT NULL," - " resolution_json TEXT," - " review_json TEXT," - " withdrawals_json TEXT," - " withdrawals_tag TEXT," - " pinned_at TEXT NOT NULL," - " PRIMARY KEY (slug, repo)," - " FOREIGN KEY (slug) REFERENCES notebooks(slug) " - " ON DELETE CASCADE" - ")" - ) - conn.execute("PRAGMA user_version = 6") - conn.execute("COMMIT") - except Exception: - conn.execute("ROLLBACK") - raise + _migrate( + conn, + 6, + "CREATE TABLE IF NOT EXISTS formal_releases (" + " slug TEXT NOT NULL," + " repo TEXT NOT NULL," + " tag TEXT NOT NULL," + " tag_object_sha TEXT NOT NULL," + " commit_sha TEXT NOT NULL," + " registry_id TEXT NOT NULL," + " registry_sha256 TEXT NOT NULL," + " env_digest TEXT NOT NULL," + " digest_provenance TEXT NOT NULL," + " asset_dir TEXT NOT NULL DEFAULT ''," + " bundle_json TEXT NOT NULL," + " registry_json TEXT NOT NULL," + " resolution_json TEXT," + " review_json TEXT," + " withdrawals_json TEXT," + " withdrawals_tag TEXT," + " pinned_at TEXT NOT NULL," + " PRIMARY KEY (slug, repo)," + " FOREIGN KEY (slug) REFERENCES notebooks(slug) " + " ON DELETE CASCADE" + ")", + ) return conn diff --git a/server/observability/spend_constants.py b/server/observability/spend_constants.py index 00b0d38..ad4b73d 100644 --- a/server/observability/spend_constants.py +++ b/server/observability/spend_constants.py @@ -76,11 +76,8 @@ #: ISO-8601 date the constants above were last verified against #: the upstream pricing pages. The quarterly restore drill cadence #: doubles as the manual review trigger; operators should update -#: this date AND the constants together. The -#: ``tests/test_spend_constants.py::TestLastVerifiedFresh::test_last_verified_within_six_months`` -#: regression guard (F3 closure from the m10 adversary critique) -#: fails CI when this date is > 180 days old, surfacing stale -#: constants automatically. +#: this date AND the constants together. No test enforces its age: +#: a suite that turns red because the calendar moved tests nothing. LAST_VERIFIED: str = "2026-05-22" #: Sanity ceiling — no provider's per-million-token cost should diff --git a/tests/test_bm25.py b/tests/test_bm25.py index d3c258b..8455739 100644 --- a/tests/test_bm25.py +++ b/tests/test_bm25.py @@ -442,6 +442,89 @@ def test_empty_corpus_raises_value_error(self, tmp_path, monkeypatch): build_bm25_index(tmp_path / "lancedb", corpus_version=version) +# =========================================================================== +# TestScanProjection — the build reads three columns, not the vectors +# =========================================================================== + + +class _ScanRecorder: + """Delegates to a LanceDB table or query; records each scan's columns.""" + + def __init__(self, inner, scans: list[list[str]], extra_rows: int = 0): + self._inner = inner + self._scans = scans + self._extra_rows = extra_rows + + def count_rows(self, *args, **kwargs) -> int: + return self._inner.count_rows(*args, **kwargs) + self._extra_rows + + def __getattr__(self, name): + method = getattr(self._inner, name) + + def call(*args, **kwargs): + result = method(*args, **kwargs) + if name == "to_arrow": + self._scans.append(result.column_names) + return result + return _ScanRecorder(result, self._scans) + + return call + + +class TestScanProjection: + def _recorded_build(self, tmp_path, monkeypatch, *, extra_rows: int = 0): + import ingest.bm25_indexer as bm25_mod + + chunks = _curated_corpus() + version = write_chunks( + chunks, _embeddings_for(chunks, seed=4), + lancedb_path=tmp_path / "lancedb", + ) + scans: list[list[str]] = [] + real_open = bm25_mod.open_chunks_table + monkeypatch.setattr( + bm25_mod, + "open_chunks_table", + lambda *a, **kw: _ScanRecorder(real_open(*a, **kw), scans, extra_rows), + ) + root = tmp_path / "bm25" + + def build(): + build_bm25_index( + tmp_path / "lancedb", corpus_version=version, index_root=root + ) + + return build, scans, root / f"v{version}", chunks + + def test_build_scans_only_the_columns_it_indexes(self, tmp_path, monkeypatch): + build, scans, version_dir, chunks = self._recorded_build( + tmp_path, monkeypatch + ) + build() + + assert scans, "the build read no rows" + for columns in scans: + assert sorted(columns) == ["body_tokens", "chunk_id", "paper_id"], ( + f"a scan returned {columns}" + ) + chunk_ids = json.loads( + (version_dir / BM25_CHUNK_IDS_NAME).read_text(encoding="utf-8") + ) + assert sorted(chunk_ids) == sorted(c.chunk_id for c in chunks) + + def test_short_scan_builds_no_index(self, tmp_path, monkeypatch): + """The table is pinned to a version, so fewer rows than it holds + means the read failed. An index from it would silently miss + chunks, so nothing is written.""" + build, _, version_dir, _ = self._recorded_build( + tmp_path, monkeypatch, extra_rows=1 + ) + with pytest.raises(RuntimeError, match="came back short"): + build() + assert not (version_dir / BM25_INDEX_NAME).exists() + assert not (version_dir / BM25_CHUNK_IDS_NAME).exists() + + # =========================================================================== # TestStatsLogging — bm25-stats.jsonl per call # =========================================================================== diff --git a/tests/test_get_paper_projection.py b/tests/test_get_paper_projection.py new file mode 100644 index 0000000..677f553 --- /dev/null +++ b/tests/test_get_paper_projection.py @@ -0,0 +1,109 @@ +"""get_paper's chunk scan reads only the columns the reply is built from. + +The chunks table carries two 1024-float vector columns. get_paper reports a +row count, the section count and two version strings, so a scan that +returns every column moves ~14 KB per row it never looks at, for up to +10,000 rows on every call. The table here is a real LanceDB table wrapped +so the test can see which columns each scan returned. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any + +import lancedb +import pytest + +from server.handlers.paper import handle_get_paper +from server.tools import reset_resources_for_tests, set_resources +from tests._graph_helpers import build_synthetic_lancedb + +_PAPER = "2401.00001" + + +class _RecordingQuery: + """Delegates to a LanceDB query builder; records each scan's columns.""" + + def __init__(self, inner: Any, scans: list[list[str]]) -> None: + self._inner = inner + self._scans = scans + + def __getattr__(self, name: str) -> Any: + method = getattr(self._inner, name) + + def call(*args: Any, **kwargs: Any) -> Any: + result = method(*args, **kwargs) + if name == "to_arrow": + self._scans.append(result.column_names) + return result + self._inner = result + return self + + return call + + +class _RecordingTable: + def __init__(self, inner: Any) -> None: + self._inner = inner + self.scans: list[list[str]] = [] + + def search(self, *args: Any, **kwargs: Any) -> _RecordingQuery: + return _RecordingQuery(self._inner.search(*args, **kwargs), self.scans) + + +@pytest.fixture +def chunks(tmp_path: Path): + rows = [ + {"chunk_id": f"arxiv:{_PAPER}:{i:016x}", "paper_id": _PAPER, + "kind": "stmt", "section_path": [section, "sub"], + "chunker_version": "cv-7", "embedder_version": "ev-3", + "embedding_stmt": [0.5] * 1024} + for i, section in enumerate(["1 Intro", "2 Main", "2 Main"]) + ] + rows.append( + {"chunk_id": "arxiv:2401.00002:0000000000000000", + "paper_id": "2401.00002", "kind": "stmt", "section_path": ["9 Other"], + "embedding_stmt": [0.5] * 1024} + ) + path = build_synthetic_lancedb(tmp_path / "lancedb", rows) + table = _RecordingTable(lancedb.connect(str(path)).open_table("chunks")) + + class _Config: + result_byte_cap = 256 * 1024 + + class _CorpusInfo: + version = 1 + + class _Resources: + config = _Config() + corpus_info = _CorpusInfo() + chunks_table = table + paper_metadata_store = None + + set_resources(_Resources()) # type: ignore[arg-type] + yield table + reset_resources_for_tests() + + +def test_reply_is_built_from_a_projected_scan(chunks: _RecordingTable) -> None: + reply = asyncio.run(handle_get_paper(paper_id=_PAPER)) + + assert reply["found"] is True + assert reply["paper"]["chunk_count"] == 3 + assert reply["paper"]["section_count"] == 2 + assert reply["paper"]["chunker_version"] == "cv-7" + assert reply["paper"]["embedder_version"] == "ev-3" + + assert len(chunks.scans) == 1 + assert sorted(chunks.scans[0]) == [ + "chunker_version", "embedder_version", "section_path", + ] + + +def test_unknown_paper_still_reports_not_found(chunks: _RecordingTable) -> None: + reply = asyncio.run(handle_get_paper(paper_id="2401.09999")) + + assert reply["found"] is False + assert reply["paper"] is None diff --git a/tests/test_handlers_chunk.py b/tests/test_handlers_chunk.py index 775b152..4a2798e 100644 --- a/tests/test_handlers_chunk.py +++ b/tests/test_handlers_chunk.py @@ -339,6 +339,56 @@ def test_aggregate_byte_bound(self, res): assert size <= 256 * 1024, f"aggregate {size} exceeds the 256 KB cap" assert all(c.get("body_truncated") for c in body["chunks"]) + def test_referenced_bodies_count_toward_the_batch_cap(self, res, monkeypatch): + """Each element's share of the cap trims only its primary body, so + 20 elements each carrying a full set of counterparts must still be + bounded as a whole: trailing entries go, the rest stay in request + order, and the reply says it was cut.""" + import json + + import server.handlers.chunk as chunk_mod + from server.proof_linkage import MAX_REFERENCED, REF_BODY_MAX_CHARS + + def _full_counterparts(table, row): + return { + "referenced_chunks": [ + { + "chunk_id": f"{row['chunk_id'][:-1]}{k}", + "kind": "proof", + "theorem_label": None, + "theorem_name": None, + "section_path": ["Chapter 1"], + "body_text": "p" * REF_BODY_MAX_CHARS, + } + for k in range(MAX_REFERENCED) + ], + "linkage": { + "direction": "statement_to_proof", + "match_basis": "theorem_label", + "outcome": "resolved", + }, + } + + monkeypatch.setattr(chunk_mod, "resolve_referenced", _full_counterparts) + ids = [f"arxiv:2401.{i:05d}:0123456789abcdef" for i in range(20)] + arrow = _chunk_arrow_multi(ids) + cols = {n: arrow.column(n).to_pylist() for n in arrow.column_names} + cols["body_text"] = ["s" * 4000] * 20 + res["holder"]["arrow"] = pa.table(cols) + + out = _run(handle_get_chunk(chunk_id=ids, include_referenced=True)) + body = out.get("structuredContent", out) + + size = len(json.dumps(body).encode()) + assert size <= 256 * 1024, f"aggregate {size} exceeds the 256 KB cap" + assert body["body_truncated"] is True + kept = [c["chunk"]["chunk_id"] for c in body["chunks"]] + assert 0 < len(kept) < len(ids) + assert kept == ids[: len(kept)] + assert all( + len(c["referenced_chunks"]) == MAX_REFERENCED for c in body["chunks"] + ) + def test_batch_is_one_query(self, res, monkeypatch): """N ids => ONE LanceDB search, not N.""" calls = {"n": 0} diff --git a/tests/test_notebooks_migration_atomicity.py b/tests/test_notebooks_migration_atomicity.py new file mode 100644 index 0000000..a4aca94 --- /dev/null +++ b/tests/test_notebooks_migration_atomicity.py @@ -0,0 +1,99 @@ +"""A notebooks.db migration step either lands whole or not at all. + +The store's connection is autocommit, so a step whose statements are not +wrapped in one transaction can stop half-applied: a column present while +``user_version`` still names the old schema. The next open then re-runs the +step, hits ``duplicate column name`` and the server cannot start. These tests +stop a step partway and check what the following open sees. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from server import notebooks_store +from server.notebooks_store import SCHEMA_VERSION, open_sync + + +def _version(db_path: Path) -> int: + conn = sqlite3.connect(str(db_path)) + try: + return int(conn.execute("PRAGMA user_version").fetchone()[0]) + finally: + conn.close() + + +def _notebook_columns(db_path: Path) -> set[str]: + conn = sqlite3.connect(str(db_path)) + try: + return { + row[1] for row in conn.execute("PRAGMA table_info(notebooks)") + } + finally: + conn.close() + + +def test_a_step_that_fails_midway_rolls_back_and_the_next_open_finishes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + db_path = tmp_path / "notebooks.db" + opened: list[sqlite3.Connection] = [] + real_connect = sqlite3.connect + + class _FailsOnSecondParseColumn(sqlite3.Connection): + """Fails the v3 -> v4 step after its first ALTER has run.""" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + opened.append(self) + + def execute(self, sql, *args): + if "ADD COLUMN parse_error" in sql: + raise sqlite3.OperationalError("injected: process died here") + return super().execute(sql, *args) + + with monkeypatch.context() as m: + m.setattr( + notebooks_store.sqlite3, + "connect", + lambda *a, **kw: real_connect( + *a, factory=_FailsOnSecondParseColumn, **kw + ), + ) + try: + with pytest.raises(sqlite3.OperationalError, match="injected"): + open_sync(db_path) + finally: + for conn in opened: + conn.close() + + # The failed step left nothing behind: still v3, and none of the + # step's columns, including the one whose ALTER did run. + assert _version(db_path) == 3 + columns = _notebook_columns(db_path) + assert "notebook_kind" in columns + assert not {"parse_status", "parse_error", "parsed_html_path"} & columns + + # A notebook written while the file sits at v3 survives the retry. + conn = sqlite3.connect(str(db_path)) + try: + conn.execute( + "INSERT INTO notebooks (slug, display_name, lancedb_path, " + "created_at) VALUES ('kept', 'Kept', '/x', '2026-01-01')" + ) + conn.commit() + finally: + conn.close() + + conn = open_sync(db_path) + try: + row = conn.execute( + "SELECT slug, parse_status, discovery_category FROM notebooks" + ).fetchall() + finally: + conn.close() + assert row == [("kept", "skipped", "")] + assert _version(db_path) == SCHEMA_VERSION diff --git a/tests/test_restore_drill.py b/tests/test_restore_drill.py index 4702ae0..1455faf 100644 --- a/tests/test_restore_drill.py +++ b/tests/test_restore_drill.py @@ -21,6 +21,7 @@ smoke_check_notebooks, write_pass_sentinel, ) +from tests._graph_helpers import build_synthetic_kuzu_graph REPO_ROOT = Path(__file__).resolve().parent.parent @@ -214,6 +215,13 @@ def test_returns_none_when_kuzu_dir_absent(self, tmp_path): # No kuzu directory at all. assert smoke_check_kuzu(tmp_path) is None + def test_counts_papers_in_a_restored_graph(self, tmp_path): + # A real graph, written where ingest writes it, under a restic-style + # absolute-path prefix. + db_path = tmp_path / "home/op/arXMCP/var/arxmcp/index/kuzu" + build_synthetic_kuzu_graph(db_path, n_papers=7) + assert smoke_check_kuzu(tmp_path) == 7 + # --------------------------------------------------------------------------- # run_check end-to-end diff --git a/tests/test_spend_constants.py b/tests/test_spend_constants.py index e6b25f5..fedc26c 100644 --- a/tests/test_spend_constants.py +++ b/tests/test_spend_constants.py @@ -4,7 +4,7 @@ - TestCostConstants Values are positive floats under the sanity ceiling. -- TestLastVerifiedFresh +- TestLastVerifiedFormat LAST_VERIFIED is parseable as YYYY-MM-DD. - TestMetricRegistration API_SPEND_USD_TOTAL is a Counter with the 3 expected labels. @@ -116,7 +116,7 @@ def test_haiku_output_more_expensive_than_input(self) -> None: ) -class TestLastVerifiedFresh: +class TestLastVerifiedFormat: def test_last_verified_parses_as_iso_date(self) -> None: try: date.fromisoformat(LAST_VERIFIED) @@ -126,42 +126,6 @@ def test_last_verified_parses_as_iso_date(self) -> None: f"YYYY-MM-DD; got: {e}" ) - def test_last_verified_within_six_months(self) -> None: - """F3 closure (m10 adversary critique): the spend_constants - docstring claimed this test existed but it didn't. Now it - does. Pricing constants go stale silently otherwise — - operators trusted LAST_VERIFIED to be enforced, but without - this guard a quarterly review can slip into ten years. - - Bound: 180 days (6 months) matches the project's quarterly - restore-drill cadence with one cycle of grace. When this - test fails, the action is: - - 1. Open the pricing pages (URLs in spend_constants.py - docstring). - 2. Update the cost constants if pricing changed. - 3. Bump LAST_VERIFIED to today. - - If you genuinely want to suppress the test for a long-paused - deployment, set ARXMCP_SKIP_STALENESS_CHECK=1 in the env — - but the spirit is that the constants get reviewed.""" - import os - - if os.getenv("ARXMCP_SKIP_STALENESS_CHECK") == "1": - pytest.skip("ARXMCP_SKIP_STALENESS_CHECK=1 set in env") - from datetime import date as _date - - verified = _date.fromisoformat(LAST_VERIFIED) - age_days = (_date.today() - verified).days - assert age_days <= 180, ( - f"spend_constants.LAST_VERIFIED is {age_days} days old " - f"(> 180 day bound). Re-verify Voyage + Anthropic " - f"pricing against the URLs in the spend_constants.py " - f"docstring, update the cost constants if changed, and " - f"bump LAST_VERIFIED to today's date. To suppress this " - f"check temporarily, set ARXMCP_SKIP_STALENESS_CHECK=1." - ) - # --------------------------------------------------------------------------- # Metric registration