From 8c8480b37abde47a25db5bffd551b67af74c872a Mon Sep 17 00:00:00 2001 From: Chris Dare Date: Sat, 14 Nov 2026 18:35:34 -0500 Subject: [PATCH 1/6] fix(tests): drop the calendar-driven spend age check test_last_verified_within_six_months failed once LAST_VERIFIED was more than 180 days old, so the default suite would turn red on 2026-11-19 with no code change. A test that fails because the calendar moved tests nothing, and the only way to make it pass is to bump the date without checking any pricing page. The age assertion is removed; the ISO-format check stays. The comment on LAST_VERIFIED no longer claims a test enforces its age. record_spend still has no caller: the Voyage call site in server/query_encoder.py is a commented-out TODO, so arxmcp_api_spend_usd_total reads 0. The metric is left as it is. Closes #326 --- server/observability/spend_constants.py | 7 ++--- tests/test_spend_constants.py | 40 ++----------------------- 2 files changed, 4 insertions(+), 43 deletions(-) diff --git a/server/observability/spend_constants.py b/server/observability/spend_constants.py index 00b0d387..ad4b73d4 100644 --- a/server/observability/spend_constants.py +++ b/server/observability/spend_constants.py @@ -76,11 +76,8 @@ #: ISO-8601 date the constants above were last verified against #: the upstream pricing pages. The quarterly restore drill cadence #: doubles as the manual review trigger; operators should update -#: this date AND the constants together. The -#: ``tests/test_spend_constants.py::TestLastVerifiedFresh::test_last_verified_within_six_months`` -#: regression guard (F3 closure from the m10 adversary critique) -#: fails CI when this date is > 180 days old, surfacing stale -#: constants automatically. +#: this date AND the constants together. No test enforces its age: +#: a suite that turns red because the calendar moved tests nothing. LAST_VERIFIED: str = "2026-05-22" #: Sanity ceiling — no provider's per-million-token cost should diff --git a/tests/test_spend_constants.py b/tests/test_spend_constants.py index e6b25f5a..fedc26cc 100644 --- a/tests/test_spend_constants.py +++ b/tests/test_spend_constants.py @@ -4,7 +4,7 @@ - TestCostConstants Values are positive floats under the sanity ceiling. -- TestLastVerifiedFresh +- TestLastVerifiedFormat LAST_VERIFIED is parseable as YYYY-MM-DD. - TestMetricRegistration API_SPEND_USD_TOTAL is a Counter with the 3 expected labels. @@ -116,7 +116,7 @@ def test_haiku_output_more_expensive_than_input(self) -> None: ) -class TestLastVerifiedFresh: +class TestLastVerifiedFormat: def test_last_verified_parses_as_iso_date(self) -> None: try: date.fromisoformat(LAST_VERIFIED) @@ -126,42 +126,6 @@ def test_last_verified_parses_as_iso_date(self) -> None: f"YYYY-MM-DD; got: {e}" ) - def test_last_verified_within_six_months(self) -> None: - """F3 closure (m10 adversary critique): the spend_constants - docstring claimed this test existed but it didn't. Now it - does. Pricing constants go stale silently otherwise — - operators trusted LAST_VERIFIED to be enforced, but without - this guard a quarterly review can slip into ten years. - - Bound: 180 days (6 months) matches the project's quarterly - restore-drill cadence with one cycle of grace. When this - test fails, the action is: - - 1. Open the pricing pages (URLs in spend_constants.py - docstring). - 2. Update the cost constants if pricing changed. - 3. Bump LAST_VERIFIED to today. - - If you genuinely want to suppress the test for a long-paused - deployment, set ARXMCP_SKIP_STALENESS_CHECK=1 in the env — - but the spirit is that the constants get reviewed.""" - import os - - if os.getenv("ARXMCP_SKIP_STALENESS_CHECK") == "1": - pytest.skip("ARXMCP_SKIP_STALENESS_CHECK=1 set in env") - from datetime import date as _date - - verified = _date.fromisoformat(LAST_VERIFIED) - age_days = (_date.today() - verified).days - assert age_days <= 180, ( - f"spend_constants.LAST_VERIFIED is {age_days} days old " - f"(> 180 day bound). Re-verify Voyage + Anthropic " - f"pricing against the URLs in the spend_constants.py " - f"docstring, update the cost constants if changed, and " - f"bump LAST_VERIFIED to today's date. To suppress this " - f"check temporarily, set ARXMCP_SKIP_STALENESS_CHECK=1." - ) - # --------------------------------------------------------------------------- # Metric registration From b8f982a11be126e21d6ada04436e3d393faa67e3 Mon Sep 17 00:00:00 2001 From: Chris Dare Date: Sat, 14 Nov 2026 18:37:02 -0500 Subject: [PATCH 2/6] fix(ops): let the restore drill read a real citation graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The drill's Kùzu check could not pass on a restored graph, for two reasons: - Kùzu 0.11 stores the database as a single file at index/kuzu, but the locator accepted only a directory there. A real graph was reported as absent and the check was skipped. - Past the locator, the count queried MATCH (p:Paper). The node table is papers, so Kùzu raised "Table Paper does not exist" and the drill reported the graph as unreadable and exited 1. The locator now takes index/kuzu whether it is a file or a directory, and the count reads papers. The new test builds a real graph with the ingest schema under a restic-style path prefix and checks the count; before this only the absent case was tested. The issue's claim about WAL checkpoints for the other SQLite stores is left for a separate change. Closes #302 --- ops/restore_drill_check.py | 28 +++++++++++++--------------- tests/test_restore_drill.py | 8 ++++++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/ops/restore_drill_check.py b/ops/restore_drill_check.py index 327fd3aa..ea15d9d0 100644 --- a/ops/restore_drill_check.py +++ b/ops/restore_drill_check.py @@ -60,18 +60,16 @@ def _locate_lancedb_root(restore_path: Path) -> Path | None: def _locate_kuzu_root(restore_path: Path) -> Path | None: """Find the restored Kùzu graph by searching for the canonical ``var/arxmcp/index/kuzu`` segment under - ``restore_path``. Returns ``None`` if no Kùzu directory + ``restore_path``. Returns ``None`` if no Kùzu graph was captured. - - Closes adversary F1 (Kùzu half). """ - # Kùzu doesn't have a single canonical marker file, so we - # look for the directory name. Filter to directories whose - # immediate parent is the expected ``index/`` so we don't - # match arbitrary "kuzu" directories the operator may have - # added under the snapshot. + # Kùzu 0.11 stores the database as a single file at this path, so + # match the name whether it is a file or a directory. Filter to + # entries whose immediate parent is the expected ``index/`` so we + # don't match arbitrary "kuzu" paths the operator may have added + # under the snapshot. for entry in restore_path.rglob("kuzu"): - if entry.is_dir() and entry.parent.name == "index": + if entry.parent.name == "index": return entry return None @@ -118,17 +116,17 @@ def smoke_check_lancedb(restore_path: Path) -> int: def smoke_check_kuzu(restore_path: Path) -> int | None: """Open the restored Kùzu graph and return paper count. - Returns ``None`` if the Kùzu directory is absent (acceptable + Returns ``None`` if the Kùzu graph is absent (acceptable — some restore paths may exclude the citation graph). Raises - ``RuntimeError`` if the directory exists but is unreadable. + ``RuntimeError`` if the graph exists but is unreadable. - Closes adversary F1 (Kùzu half): finds the Kùzu directory - via rglob rather than hardcoding the path prefix. + Finds the graph via rglob rather than hardcoding the path + prefix, because restic restores under the absolute source path. """ kuzu_path = _locate_kuzu_root(restore_path) if kuzu_path is None: logger.info( - "restore_drill: Kùzu directory absent under %s — skipping", + "restore_drill: Kùzu graph absent under %s — skipping", restore_path, ) return None @@ -145,7 +143,7 @@ def smoke_check_kuzu(restore_path: Path) -> int | None: db = kuzu.Database(str(kuzu_path), read_only=True) conn = kuzu.Connection(db) # Restored DB may be empty; we only require it to open. - result = conn.execute("MATCH (p:Paper) RETURN count(p) AS c") + result = conn.execute("MATCH (p:papers) RETURN count(p) AS c") paper_count = 0 while result.has_next(): row = result.get_next() diff --git a/tests/test_restore_drill.py b/tests/test_restore_drill.py index 4702ae0e..1455faf8 100644 --- a/tests/test_restore_drill.py +++ b/tests/test_restore_drill.py @@ -21,6 +21,7 @@ smoke_check_notebooks, write_pass_sentinel, ) +from tests._graph_helpers import build_synthetic_kuzu_graph REPO_ROOT = Path(__file__).resolve().parent.parent @@ -214,6 +215,13 @@ def test_returns_none_when_kuzu_dir_absent(self, tmp_path): # No kuzu directory at all. assert smoke_check_kuzu(tmp_path) is None + def test_counts_papers_in_a_restored_graph(self, tmp_path): + # A real graph, written where ingest writes it, under a restic-style + # absolute-path prefix. + db_path = tmp_path / "home/op/arXMCP/var/arxmcp/index/kuzu" + build_synthetic_kuzu_graph(db_path, n_papers=7) + assert smoke_check_kuzu(tmp_path) == 7 + # --------------------------------------------------------------------------- # run_check end-to-end From 2afb88dc932b4d00b3bdb0a4d965a921e23d2bbc Mon Sep 17 00:00:00 2001 From: Chris Dare Date: Sat, 14 Nov 2026 18:39:02 -0500 Subject: [PATCH 3/6] fix(server): make every notebooks.db migration step atomic The connection is autocommit, and the v2->v3 and v3->v4 steps ran their ALTERs and the user_version bump as separate statements. A crash between them left a column in place while user_version still named the old schema, so the next open re-ran the step, hit "duplicate column name", and the server could not start. v3->v4 has three ALTERs, so its window was the widest. Only v4->v5 and v5->v6 opened a transaction. Every step now goes through one helper, _migrate, which runs the statements and the version bump between BEGIN and COMMIT and rolls back on any error, including KeyboardInterrupt. The v4->v5 comment that explained this in one place moves to the helper. The new test fails the second v3->v4 ALTER, then checks that the file is still at v3 with none of that step's columns, and that the next open migrates to the current version and keeps a notebook written in between. Closes #299 --- server/notebooks_store.py | 170 ++++++++++---------- tests/test_notebooks_migration_atomicity.py | 99 ++++++++++++ 2 files changed, 182 insertions(+), 87 deletions(-) create mode 100644 tests/test_notebooks_migration_atomicity.py diff --git a/server/notebooks_store.py b/server/notebooks_store.py index ed970e20..367bedc5 100644 --- a/server/notebooks_store.py +++ b/server/notebooks_store.py @@ -89,6 +89,32 @@ SCHEMA_VERSION: int = 6 +def _migrate(conn: sqlite3.Connection, version: int, *statements: str) -> None: + """Run one migration step and stamp ``user_version`` atomically. + + The connection is ``isolation_level=None`` (autocommit), so without an + explicit BEGIN every statement commits on its own. A crash or error + between two of a step's statements would leave the DB half-migrated + (``user_version`` still the old value, some columns already present), + so the next open re-runs the step and its first ALTER raises + ``sqlite3.OperationalError: duplicate column name``, crash-looping the + daemon at startup. SQLite DDL and the ``user_version`` write are + transactional, so BEGIN/COMMIT makes the whole step all-or-nothing. + """ + conn.execute("BEGIN") + try: + for statement in statements: + conn.execute(statement) + conn.execute(f"PRAGMA user_version = {int(version)}") + conn.execute("COMMIT") + except BaseException: + # SQLite rolls some errors back on its own (SQLITE_FULL, for one); + # a second ROLLBACK would raise and hide the original error. + if conn.in_transaction: + conn.execute("ROLLBACK") + raise + + #: Module-level so a SYNC CLI can reach the migration ladder. #: #: It used to be a closure inside ``NotebooksStore.open``, which meant the @@ -154,17 +180,17 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # EXISTS) plus the preceding DROP IF EXISTS preserves # the original safety on empty DBs. if current_version < 1: - conn.execute("DROP TABLE IF EXISTS notebook_papers") - conn.execute("DROP TABLE IF EXISTS notebooks") - conn.execute( + _migrate( + conn, + 1, + "DROP TABLE IF EXISTS notebook_papers", + "DROP TABLE IF EXISTS notebooks", "CREATE TABLE notebooks (" " slug TEXT PRIMARY KEY," " display_name TEXT NOT NULL DEFAULT ''," " lancedb_path TEXT NOT NULL," " created_at TEXT NOT NULL" - ")" - ) - conn.execute( + ")", "CREATE TABLE notebook_papers (" " slug TEXT NOT NULL," " paper_id TEXT NOT NULL," @@ -172,13 +198,10 @@ def open_sync(db_path: Path) -> sqlite3.Connection: " PRIMARY KEY (slug, paper_id)," " FOREIGN KEY (slug) REFERENCES notebooks(slug) " " ON DELETE CASCADE" - ")" - ) - conn.execute( + ")", "CREATE INDEX idx_notebook_papers_slug " - "ON notebook_papers(slug)" + "ON notebook_papers(slug)", ) - conn.execute("PRAGMA user_version = 1") # v1 -> v2: m9 ADDITIVE migration. notebook_ingest_runs # stores per-ingest-trigger run state. CREATE TABLE IF # NOT EXISTS (NOT a destructive recreate) preserves all @@ -187,7 +210,9 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # pattern is acceptable for caches (miss, not data # loss) but NOT for notebook metadata. if current_version < 2: - conn.execute( + _migrate( + conn, + 2, "CREATE TABLE IF NOT EXISTS notebook_ingest_runs (" " id INTEGER PRIMARY KEY AUTOINCREMENT," " slug TEXT NOT NULL," @@ -198,13 +223,10 @@ def open_sync(db_path: Path) -> sqlite3.Connection: " stderr_tail TEXT," " FOREIGN KEY (slug) REFERENCES notebooks(slug) " " ON DELETE CASCADE" - ")" - ) - conn.execute( + ")", "CREATE INDEX IF NOT EXISTS idx_runs_slug " - "ON notebook_ingest_runs(slug, id DESC)" + "ON notebook_ingest_runs(slug, id DESC)", ) - conn.execute("PRAGMA user_version = 2") # v2 -> v3: textbook-ingest-m3 ADDITIVE migration. # ``notebook_kind`` distinguishes arXiv-corpus notebooks # (the historical default) from textbook-corpus notebooks @@ -217,11 +239,12 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # ``NotebookCreate.notebook_kind`` field enforces the # ``{arxiv, textbook}`` enum domain at the write path. if current_version < 3: - conn.execute( + _migrate( + conn, + 3, "ALTER TABLE notebooks ADD COLUMN notebook_kind " - "TEXT NOT NULL DEFAULT 'arxiv'" + "TEXT NOT NULL DEFAULT 'arxiv'", ) - conn.execute("PRAGMA user_version = 3") # v3 -> v4: textbook-ingest-m6 ADDITIVE migration. Three # parse-tracking columns on the ``notebooks`` table: # @@ -238,19 +261,16 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # rendered ``index.html``. Empty until the parse task # reports complete. if current_version < 4: - conn.execute( + _migrate( + conn, + 4, "ALTER TABLE notebooks ADD COLUMN parse_status " - "TEXT NOT NULL DEFAULT 'skipped'" - ) - conn.execute( + "TEXT NOT NULL DEFAULT 'skipped'", "ALTER TABLE notebooks ADD COLUMN parse_error " - "TEXT NOT NULL DEFAULT ''" - ) - conn.execute( + "TEXT NOT NULL DEFAULT ''", "ALTER TABLE notebooks ADD COLUMN parsed_html_path " - "TEXT NOT NULL DEFAULT ''" + "TEXT NOT NULL DEFAULT ''", ) - conn.execute("PRAGMA user_version = 4") # v4 -> v5: notebook-paper-discovery-m1 ADDITIVE migration. # Two columns on ``notebooks`` give a notebook a machine- # readable research interest for topic-driven paper discovery @@ -259,34 +279,15 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # {math.AG, math.NT, math-ph, hep-th}) and free-text # ``description``. Column-level DEFAULT '' backfills every # existing row in O(1) (no row-rewrite). - # - # Unlike the v1->v4 blocks (each a single ALTER), v4->v5 adds - # TWO columns. A crash BETWEEN the two ALTERs would leave the - # DB half-migrated (``user_version`` still 4 but - # ``discovery_category`` already present), so the next open - # re-runs the block and the first ALTER raises - # ``sqlite3.OperationalError: duplicate column name``, - # crash-looping the daemon at startup. Wrap the two ALTERs + - # the version bump in an explicit BEGIN/COMMIT so the whole - # step is atomic and re-runnable. The connection is - # ``isolation_level=None`` (autocommit), so the transaction - # MUST be opened explicitly. if current_version < 5: - conn.execute("BEGIN") - try: - conn.execute( - "ALTER TABLE notebooks ADD COLUMN " - "discovery_category TEXT NOT NULL DEFAULT ''" - ) - conn.execute( - "ALTER TABLE notebooks ADD COLUMN " - "description TEXT NOT NULL DEFAULT ''" - ) - conn.execute("PRAGMA user_version = 5") - conn.execute("COMMIT") - except Exception: - conn.execute("ROLLBACK") - raise + _migrate( + conn, + 5, + "ALTER TABLE notebooks ADD COLUMN " + "discovery_category TEXT NOT NULL DEFAULT ''", + "ALTER TABLE notebooks ADD COLUMN " + "description TEXT NOT NULL DEFAULT ''", + ) # v5 -> v6: contract-v1 ADDITIVE migration. # ``formal_releases`` is arXMCP's PIN to a topic repo's # released formalization. ADR-0004: arXMCP hosts no @@ -308,37 +309,32 @@ def open_sync(db_path: Path) -> sqlite3.Connection: # and 4.9 rule 1 forbids collapsing that into a pass/fail # token. See ``tools/formal_release_pin.py``. if current_version < 6: - conn.execute("BEGIN") - try: - conn.execute( - "CREATE TABLE IF NOT EXISTS formal_releases (" - " slug TEXT NOT NULL," - " repo TEXT NOT NULL," - " tag TEXT NOT NULL," - " tag_object_sha TEXT NOT NULL," - " commit_sha TEXT NOT NULL," - " registry_id TEXT NOT NULL," - " registry_sha256 TEXT NOT NULL," - " env_digest TEXT NOT NULL," - " digest_provenance TEXT NOT NULL," - " asset_dir TEXT NOT NULL DEFAULT ''," - " bundle_json TEXT NOT NULL," - " registry_json TEXT NOT NULL," - " resolution_json TEXT," - " review_json TEXT," - " withdrawals_json TEXT," - " withdrawals_tag TEXT," - " pinned_at TEXT NOT NULL," - " PRIMARY KEY (slug, repo)," - " FOREIGN KEY (slug) REFERENCES notebooks(slug) " - " ON DELETE CASCADE" - ")" - ) - conn.execute("PRAGMA user_version = 6") - conn.execute("COMMIT") - except Exception: - conn.execute("ROLLBACK") - raise + _migrate( + conn, + 6, + "CREATE TABLE IF NOT EXISTS formal_releases (" + " slug TEXT NOT NULL," + " repo TEXT NOT NULL," + " tag TEXT NOT NULL," + " tag_object_sha TEXT NOT NULL," + " commit_sha TEXT NOT NULL," + " registry_id TEXT NOT NULL," + " registry_sha256 TEXT NOT NULL," + " env_digest TEXT NOT NULL," + " digest_provenance TEXT NOT NULL," + " asset_dir TEXT NOT NULL DEFAULT ''," + " bundle_json TEXT NOT NULL," + " registry_json TEXT NOT NULL," + " resolution_json TEXT," + " review_json TEXT," + " withdrawals_json TEXT," + " withdrawals_tag TEXT," + " pinned_at TEXT NOT NULL," + " PRIMARY KEY (slug, repo)," + " FOREIGN KEY (slug) REFERENCES notebooks(slug) " + " ON DELETE CASCADE" + ")", + ) return conn diff --git a/tests/test_notebooks_migration_atomicity.py b/tests/test_notebooks_migration_atomicity.py new file mode 100644 index 00000000..a4aca94c --- /dev/null +++ b/tests/test_notebooks_migration_atomicity.py @@ -0,0 +1,99 @@ +"""A notebooks.db migration step either lands whole or not at all. + +The store's connection is autocommit, so a step whose statements are not +wrapped in one transaction can stop half-applied: a column present while +``user_version`` still names the old schema. The next open then re-runs the +step, hits ``duplicate column name`` and the server cannot start. These tests +stop a step partway and check what the following open sees. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from server import notebooks_store +from server.notebooks_store import SCHEMA_VERSION, open_sync + + +def _version(db_path: Path) -> int: + conn = sqlite3.connect(str(db_path)) + try: + return int(conn.execute("PRAGMA user_version").fetchone()[0]) + finally: + conn.close() + + +def _notebook_columns(db_path: Path) -> set[str]: + conn = sqlite3.connect(str(db_path)) + try: + return { + row[1] for row in conn.execute("PRAGMA table_info(notebooks)") + } + finally: + conn.close() + + +def test_a_step_that_fails_midway_rolls_back_and_the_next_open_finishes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + db_path = tmp_path / "notebooks.db" + opened: list[sqlite3.Connection] = [] + real_connect = sqlite3.connect + + class _FailsOnSecondParseColumn(sqlite3.Connection): + """Fails the v3 -> v4 step after its first ALTER has run.""" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + opened.append(self) + + def execute(self, sql, *args): + if "ADD COLUMN parse_error" in sql: + raise sqlite3.OperationalError("injected: process died here") + return super().execute(sql, *args) + + with monkeypatch.context() as m: + m.setattr( + notebooks_store.sqlite3, + "connect", + lambda *a, **kw: real_connect( + *a, factory=_FailsOnSecondParseColumn, **kw + ), + ) + try: + with pytest.raises(sqlite3.OperationalError, match="injected"): + open_sync(db_path) + finally: + for conn in opened: + conn.close() + + # The failed step left nothing behind: still v3, and none of the + # step's columns, including the one whose ALTER did run. + assert _version(db_path) == 3 + columns = _notebook_columns(db_path) + assert "notebook_kind" in columns + assert not {"parse_status", "parse_error", "parsed_html_path"} & columns + + # A notebook written while the file sits at v3 survives the retry. + conn = sqlite3.connect(str(db_path)) + try: + conn.execute( + "INSERT INTO notebooks (slug, display_name, lancedb_path, " + "created_at) VALUES ('kept', 'Kept', '/x', '2026-01-01')" + ) + conn.commit() + finally: + conn.close() + + conn = open_sync(db_path) + try: + row = conn.execute( + "SELECT slug, parse_status, discovery_category FROM notebooks" + ).fetchall() + finally: + conn.close() + assert row == [("kept", "skipped", "")] + assert _version(db_path) == SCHEMA_VERSION From 59ccc2ac460254edc7c9f0a0bac3579ee0f54143 Mon Sep 17 00:00:00 2001 From: Chris Dare Date: Sat, 14 Nov 2026 18:40:43 -0500 Subject: [PATCH 4/6] fix(server): project get_paper's chunk scan to three columns get_paper read up to 10,000 chunk rows with every column, including both 1024-float vector columns (~14 KB per row), and then used only section_path, chunker_version, embedder_version and the row count. That ran on every call, on the event loop. The scan now selects those three columns at the scanner, the same pattern as the lemma fallback and the notebook recount. The reply is unchanged. The new test wraps a real LanceDB table so it can see the columns each scan returned. It checks the reply's counts and versions and that the scan carried exactly the three columns; before the fix it returned all 26. Closes #353 --- server/handlers/paper.py | 8 ++- tests/test_get_paper_projection.py | 109 +++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 tests/test_get_paper_projection.py diff --git a/server/handlers/paper.py b/server/handlers/paper.py index 72f30862..4d930a74 100644 --- a/server/handlers/paper.py +++ b/server/handlers/paper.py @@ -57,6 +57,9 @@ logger = logging.getLogger(__name__) +#: The only chunk columns ``get_paper`` reads; the row count comes free. +_PAPER_CHUNK_COLUMNS = ["section_path", "chunker_version", "embedder_version"] + def hydrated_paper_fields(record: Any) -> dict[str, Any] | None: """Map a ``PaperMetadataRecord`` to the ``get_paper`` identity @@ -164,10 +167,13 @@ async def handle_get_paper( # Filter to rows matching paper_id. LanceDB doesn't expose a # native group-by; pull the matching rows in Arrow and aggregate # in Python. For the 50-paper corpus the worst case is ~100s of - # rows per paper — fast enough. + # rows per paper — fast enough. Project at the scanner: an + # unprojected read pulls both 1024-float vector columns (~14 KB/row) + # for every row only to count them and read two version strings. arrow = ( r.chunks_table.search() .where(f"paper_id = '{_escape(paper_id)}'", prefilter=True) + .select(_PAPER_CHUNK_COLUMNS) .limit(10000) .to_arrow() ) diff --git a/tests/test_get_paper_projection.py b/tests/test_get_paper_projection.py new file mode 100644 index 00000000..677f5538 --- /dev/null +++ b/tests/test_get_paper_projection.py @@ -0,0 +1,109 @@ +"""get_paper's chunk scan reads only the columns the reply is built from. + +The chunks table carries two 1024-float vector columns. get_paper reports a +row count, the section count and two version strings, so a scan that +returns every column moves ~14 KB per row it never looks at, for up to +10,000 rows on every call. The table here is a real LanceDB table wrapped +so the test can see which columns each scan returned. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any + +import lancedb +import pytest + +from server.handlers.paper import handle_get_paper +from server.tools import reset_resources_for_tests, set_resources +from tests._graph_helpers import build_synthetic_lancedb + +_PAPER = "2401.00001" + + +class _RecordingQuery: + """Delegates to a LanceDB query builder; records each scan's columns.""" + + def __init__(self, inner: Any, scans: list[list[str]]) -> None: + self._inner = inner + self._scans = scans + + def __getattr__(self, name: str) -> Any: + method = getattr(self._inner, name) + + def call(*args: Any, **kwargs: Any) -> Any: + result = method(*args, **kwargs) + if name == "to_arrow": + self._scans.append(result.column_names) + return result + self._inner = result + return self + + return call + + +class _RecordingTable: + def __init__(self, inner: Any) -> None: + self._inner = inner + self.scans: list[list[str]] = [] + + def search(self, *args: Any, **kwargs: Any) -> _RecordingQuery: + return _RecordingQuery(self._inner.search(*args, **kwargs), self.scans) + + +@pytest.fixture +def chunks(tmp_path: Path): + rows = [ + {"chunk_id": f"arxiv:{_PAPER}:{i:016x}", "paper_id": _PAPER, + "kind": "stmt", "section_path": [section, "sub"], + "chunker_version": "cv-7", "embedder_version": "ev-3", + "embedding_stmt": [0.5] * 1024} + for i, section in enumerate(["1 Intro", "2 Main", "2 Main"]) + ] + rows.append( + {"chunk_id": "arxiv:2401.00002:0000000000000000", + "paper_id": "2401.00002", "kind": "stmt", "section_path": ["9 Other"], + "embedding_stmt": [0.5] * 1024} + ) + path = build_synthetic_lancedb(tmp_path / "lancedb", rows) + table = _RecordingTable(lancedb.connect(str(path)).open_table("chunks")) + + class _Config: + result_byte_cap = 256 * 1024 + + class _CorpusInfo: + version = 1 + + class _Resources: + config = _Config() + corpus_info = _CorpusInfo() + chunks_table = table + paper_metadata_store = None + + set_resources(_Resources()) # type: ignore[arg-type] + yield table + reset_resources_for_tests() + + +def test_reply_is_built_from_a_projected_scan(chunks: _RecordingTable) -> None: + reply = asyncio.run(handle_get_paper(paper_id=_PAPER)) + + assert reply["found"] is True + assert reply["paper"]["chunk_count"] == 3 + assert reply["paper"]["section_count"] == 2 + assert reply["paper"]["chunker_version"] == "cv-7" + assert reply["paper"]["embedder_version"] == "ev-3" + + assert len(chunks.scans) == 1 + assert sorted(chunks.scans[0]) == [ + "chunker_version", "embedder_version", "section_path", + ] + + +def test_unknown_paper_still_reports_not_found(chunks: _RecordingTable) -> None: + reply = asyncio.run(handle_get_paper(paper_id="2401.09999")) + + assert reply["found"] is False + assert reply["paper"] is None From 3bd720cd47e0c2a6cbfe296affacdc0b3cea5657 Mon Sep 17 00:00:00 2001 From: Chris Dare Date: Sat, 14 Nov 2026 18:42:24 -0500 Subject: [PATCH 5/6] fix(server): bound a batched get_chunk reply as a whole A batched get_chunk gave each element result_byte_cap // n, but enforce_byte_cap trims only the element's primary body. Referenced counterparts (up to 8 bodies of 4000 characters each) were never cut, so 20 elements with include_referenced=True came to about 700 KB of structured content, nearly three times the 256 KB cap, with no body_truncated flag. /mcp is exempt from the response-size middleware, so nothing downstream caught it. The assembled {chunks: [...]} is now passed through cap_result_list, which drops trailing entries until the reply fits and sets body_truncated. The per-element share still trims oversized primary bodies first, so a batch of large primaries keeps every entry, as before. The new test gives each of 20 elements a full set of counterparts. It checks the reply fits the cap, is flagged, and keeps a request-order prefix of the ids. Before the fix the reply was 706,555 bytes. Closes #327 --- server/handlers/chunk.py | 37 ++++++++++++++++---------- tests/test_handlers_chunk.py | 50 ++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 14 deletions(-) diff --git a/server/handlers/chunk.py b/server/handlers/chunk.py index 360a5da5..dbd545b7 100644 --- a/server/handlers/chunk.py +++ b/server/handlers/chunk.py @@ -36,6 +36,7 @@ from server.observability.sanitize import sanitize_retrieved_text from server.proof_linkage import resolve_referenced from server.tools import ( + cap_result_list, enforce_byte_cap, envelope, get_resources, @@ -78,6 +79,9 @@ async def handle_get_chunk( ids returns ``{chunks: [, ...]}`` in request order — one entry per id, each carrying its own ``found``, byte-cap, and linkage — so N ids cost one round-trip instead of N. + If the assembled batch is still over the byte cap, trailing entries + are dropped and ``body_truncated`` is set at the top level; the ids + past the last entry can be fetched again. A malformed id raises a JSON-RPC error; a well-formed id absent from the corpus yields a per-chunk ``found: false`` (batch elements stay @@ -113,21 +117,26 @@ async def handle_get_chunk( return envelope( _build_chunk_result(ids[0], row_map.get(ids[0]), include_referenced, r) ) - # Per-element byte budget so the whole chunks[] array is bounded by - # the per-response cap, not n independent copies of it. Each element - # gets result_byte_cap // n; a chunk over its share is truncated to a - # resource_link exactly as a scalar over-cap chunk is, so every - # requested id is still returned. + # Per-element byte budget: each element gets result_byte_cap // n, + # and a primary body over its share is truncated to a resource_link + # exactly as a scalar over-cap chunk is. That share bounds only the + # primary body: referenced counterparts (up to MAX_REFERENCED bodies + # each) are never trimmed by it, so the assembled array is capped + # again as a whole, dropping trailing entries until it fits. per_elem_cap = getattr(r.config, "result_byte_cap", 256 * 1024) // len(ids) - return envelope({ - "chunks": [ - _build_chunk_result( - cid, row_map.get(cid), include_referenced, r, - byte_cap=per_elem_cap, - ) - for cid in ids - ] - }) + structured, _ = cap_result_list( + { + "chunks": [ + _build_chunk_result( + cid, row_map.get(cid), include_referenced, r, + byte_cap=per_elem_cap, + ) + for cid in ids + ] + }, + list_key="chunks", + ) + return envelope(structured) def _fetch_chunk_rows(r, ids: list[str]) -> dict[str, Any]: # noqa: ANN001 diff --git a/tests/test_handlers_chunk.py b/tests/test_handlers_chunk.py index 775b1523..4a2798ed 100644 --- a/tests/test_handlers_chunk.py +++ b/tests/test_handlers_chunk.py @@ -339,6 +339,56 @@ def test_aggregate_byte_bound(self, res): assert size <= 256 * 1024, f"aggregate {size} exceeds the 256 KB cap" assert all(c.get("body_truncated") for c in body["chunks"]) + def test_referenced_bodies_count_toward_the_batch_cap(self, res, monkeypatch): + """Each element's share of the cap trims only its primary body, so + 20 elements each carrying a full set of counterparts must still be + bounded as a whole: trailing entries go, the rest stay in request + order, and the reply says it was cut.""" + import json + + import server.handlers.chunk as chunk_mod + from server.proof_linkage import MAX_REFERENCED, REF_BODY_MAX_CHARS + + def _full_counterparts(table, row): + return { + "referenced_chunks": [ + { + "chunk_id": f"{row['chunk_id'][:-1]}{k}", + "kind": "proof", + "theorem_label": None, + "theorem_name": None, + "section_path": ["Chapter 1"], + "body_text": "p" * REF_BODY_MAX_CHARS, + } + for k in range(MAX_REFERENCED) + ], + "linkage": { + "direction": "statement_to_proof", + "match_basis": "theorem_label", + "outcome": "resolved", + }, + } + + monkeypatch.setattr(chunk_mod, "resolve_referenced", _full_counterparts) + ids = [f"arxiv:2401.{i:05d}:0123456789abcdef" for i in range(20)] + arrow = _chunk_arrow_multi(ids) + cols = {n: arrow.column(n).to_pylist() for n in arrow.column_names} + cols["body_text"] = ["s" * 4000] * 20 + res["holder"]["arrow"] = pa.table(cols) + + out = _run(handle_get_chunk(chunk_id=ids, include_referenced=True)) + body = out.get("structuredContent", out) + + size = len(json.dumps(body).encode()) + assert size <= 256 * 1024, f"aggregate {size} exceeds the 256 KB cap" + assert body["body_truncated"] is True + kept = [c["chunk"]["chunk_id"] for c in body["chunks"]] + assert 0 < len(kept) < len(ids) + assert kept == ids[: len(kept)] + assert all( + len(c["referenced_chunks"]) == MAX_REFERENCED for c in body["chunks"] + ) + def test_batch_is_one_query(self, res, monkeypatch): """N ids => ONE LanceDB search, not N.""" calls = {"n": 0} From 8885093e322fe77aace08f239ec0fd211f25a5e3 Mon Sep 17 00:00:00 2001 From: Chris Dare Date: Sat, 14 Nov 2026 18:43:34 -0500 Subject: [PATCH 6/6] fix(ingest): project the BM25 auto-build scan to three columns build_bm25_index read the pinned chunks table with tbl.to_arrow(), which in lancedb 0.30 materializes every column, both 1024-float vector columns included, and then used chunk_id, body_tokens and paper_id. On a cold start with no index this ran on the startup path. The scan now goes through project_columns with the table's row count as expected_rows. The table is checked out at corpus_version, so a short read means the read failed; the build raises instead of writing an index that silently misses chunks. The other half of the finding, the startup chunk_id cross-check in server/resources.py, already projects through project_column. Two new tests wrap the real table. One checks that every scan returned exactly the three columns and the index covers every chunk (before the fix the scan returned all 26). The other makes the scan come back one row short and checks that nothing is written. Closes #336 --- ingest/bm25_indexer.py | 21 ++++++++++- tests/test_bm25.py | 83 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/ingest/bm25_indexer.py b/ingest/bm25_indexer.py index 7560d894..9f881890 100644 --- a/ingest/bm25_indexer.py +++ b/ingest/bm25_indexer.py @@ -84,7 +84,7 @@ from rank_bm25 import BM25Okapi -from server.corpus import open_chunks_table +from server.corpus import open_chunks_table, project_columns logger = logging.getLogger(__name__) @@ -303,6 +303,9 @@ def build_bm25_index( FileNotFoundError ``lancedb_path`` does not exist on disk (re-raised from :func:`open_chunks_table`). + RuntimeError + The projected scan returned fewer rows than the pinned version + holds. """ # Closes L3 from the E04_S04 critique: defense-in-depth validation # of the version integer. ``_bm25_version_dir(-1)`` would produce @@ -352,7 +355,21 @@ def build_bm25_index( # validates the path, performs ``tbl.checkout(version)`` for MVCC # isolation, and re-raises bad-version errors as ``ValueError``. tbl = open_chunks_table(lancedb_path, version=corpus_version) - arrow = tbl.to_arrow() + # Project at the scanner: `tbl.to_arrow()` would materialize both + # 1024-float vector columns for every row to read three string + # columns. The table is checked out at `corpus_version`, so a short + # read is a failed read, and an index built from it would silently + # omit chunks; refuse instead. + arrow = project_columns( + tbl, + ["chunk_id", "body_tokens", "paper_id"], + expected_rows=tbl.count_rows(), + ) + if arrow is None: + raise RuntimeError( + f"chunks scan came back short at corpus_version={corpus_version}; " + "refusing to build a BM25 index from a partial read" + ) raw_chunk_ids = arrow.column("chunk_id").to_pylist() raw_body_tokens = arrow.column("body_tokens").to_pylist() raw_paper_ids = arrow.column("paper_id").to_pylist() diff --git a/tests/test_bm25.py b/tests/test_bm25.py index d3c258b4..84557392 100644 --- a/tests/test_bm25.py +++ b/tests/test_bm25.py @@ -442,6 +442,89 @@ def test_empty_corpus_raises_value_error(self, tmp_path, monkeypatch): build_bm25_index(tmp_path / "lancedb", corpus_version=version) +# =========================================================================== +# TestScanProjection — the build reads three columns, not the vectors +# =========================================================================== + + +class _ScanRecorder: + """Delegates to a LanceDB table or query; records each scan's columns.""" + + def __init__(self, inner, scans: list[list[str]], extra_rows: int = 0): + self._inner = inner + self._scans = scans + self._extra_rows = extra_rows + + def count_rows(self, *args, **kwargs) -> int: + return self._inner.count_rows(*args, **kwargs) + self._extra_rows + + def __getattr__(self, name): + method = getattr(self._inner, name) + + def call(*args, **kwargs): + result = method(*args, **kwargs) + if name == "to_arrow": + self._scans.append(result.column_names) + return result + return _ScanRecorder(result, self._scans) + + return call + + +class TestScanProjection: + def _recorded_build(self, tmp_path, monkeypatch, *, extra_rows: int = 0): + import ingest.bm25_indexer as bm25_mod + + chunks = _curated_corpus() + version = write_chunks( + chunks, _embeddings_for(chunks, seed=4), + lancedb_path=tmp_path / "lancedb", + ) + scans: list[list[str]] = [] + real_open = bm25_mod.open_chunks_table + monkeypatch.setattr( + bm25_mod, + "open_chunks_table", + lambda *a, **kw: _ScanRecorder(real_open(*a, **kw), scans, extra_rows), + ) + root = tmp_path / "bm25" + + def build(): + build_bm25_index( + tmp_path / "lancedb", corpus_version=version, index_root=root + ) + + return build, scans, root / f"v{version}", chunks + + def test_build_scans_only_the_columns_it_indexes(self, tmp_path, monkeypatch): + build, scans, version_dir, chunks = self._recorded_build( + tmp_path, monkeypatch + ) + build() + + assert scans, "the build read no rows" + for columns in scans: + assert sorted(columns) == ["body_tokens", "chunk_id", "paper_id"], ( + f"a scan returned {columns}" + ) + chunk_ids = json.loads( + (version_dir / BM25_CHUNK_IDS_NAME).read_text(encoding="utf-8") + ) + assert sorted(chunk_ids) == sorted(c.chunk_id for c in chunks) + + def test_short_scan_builds_no_index(self, tmp_path, monkeypatch): + """The table is pinned to a version, so fewer rows than it holds + means the read failed. An index from it would silently miss + chunks, so nothing is written.""" + build, _, version_dir, _ = self._recorded_build( + tmp_path, monkeypatch, extra_rows=1 + ) + with pytest.raises(RuntimeError, match="came back short"): + build() + assert not (version_dir / BM25_INDEX_NAME).exists() + assert not (version_dir / BM25_CHUNK_IDS_NAME).exists() + + # =========================================================================== # TestStatsLogging — bm25-stats.jsonl per call # ===========================================================================