From d4b5f480550d9158c1540e658b5359d710068ce7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 16:05:48 +0000 Subject: [PATCH 1/3] fix(server): link only the search rows the cap keeps search_papers built one resource_link per result row from the uncapped row list, after the byte cap had already dropped trailing rows from structuredContent. When the cap fired, content[] pointed at chunks that were not in the payload it shipped with. The links are now built from structuredContent["results"], the rows that survived. The docstring said search_papers does not cap at all, though it has for a long time. It now says what the code does, including that the links are outside the cap's measurement: about 200 bytes each, ~10 KB for 50 rows against the 256 KB cap. A new test drives the handler with a cap that keeps about half the rows and expects exactly one link per kept row. The unfixed handler emits ten links for five rows. Closes #352. --- server/handlers/search.py | 33 +++++++++++++++------------------ tests/test_search_filter.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/server/handlers/search.py b/server/handlers/search.py index c3ac8038..058401a6 100644 --- a/server/handlers/search.py +++ b/server/handlers/search.py @@ -716,7 +716,7 @@ async def handle_search_papers( getattr(r, "paper_metadata_store", None), rows ) structured = _cap(structured) - content = _build_content_blocks(structured, rows) + content = _build_content_blocks(structured) return CallToolResult(content=content, structuredContent=structured) # Encode query (singleflight + semaphore — two-tier concurrency). @@ -781,7 +781,7 @@ async def handle_search_papers( getattr(r, "paper_metadata_store", None), rows ) structured = _cap(structured) - content = _build_content_blocks(structured, rows) + content = _build_content_blocks(structured) return CallToolResult(content=content, structuredContent=structured) # Dense ANN over embedding_stmt only. embedding_proof is for @@ -955,7 +955,7 @@ async def handle_search_papers( # one ResourceLink per result row. E13_S04b: cap the structured # payload (defensive; closes Threat 4 partial-coverage gap G1). structured = _cap(structured) - content = _build_content_blocks(structured, rows) + content = _build_content_blocks(structured) return CallToolResult(content=content, structuredContent=structured) @@ -1071,9 +1071,7 @@ def _cap(structured: dict[str, Any]) -> dict[str, Any]: return structured -def _build_content_blocks( - structured: dict[str, Any], rows: list[dict[str, Any]] -) -> list[Any]: +def _build_content_blocks(structured: dict[str, Any]) -> list[Any]: """Build the ``content`` array per the E06_S04 contract. Block 0 is a TextContent carrying ``json.dumps(structured, @@ -1081,17 +1079,16 @@ def _build_content_blocks( dict-handler emits, so clients that only read ``content[0].text`` see the full payload. - Blocks 1..N are ResourceLink blocks in the same order as - ``rows`` (which is already sorted ``(score_desc, - chunk_id_asc)``). Each link's URI is - ``arxmcp://chunks/`` per the design note's resource - URI scheme. - - Note: ``search_papers`` does NOT call ``enforce_byte_cap`` - (only ``get_chunk`` does at v1). When a future milestone - wires cap enforcement here, the per-row ResourceLink overhead - must be added to the wire-overhead factor calibration in - ``server/tools.py``. + Blocks 1..N are ResourceLink blocks, one per row of + ``structured["results"]`` in the same order (already sorted + ``(score_desc, chunk_id_asc)``). Each link's URI is + ``arxmcp://chunks/``, served by the chunk resource in + :mod:`server.mcp_resources`. + + The links are built from ``structured`` AFTER the byte cap, so a + row the cap dropped gets no link. The links themselves are not in + the cap's measurement: about 200 bytes each as JSON, so ~10 KB for + the 50-row maximum against a 256 KB cap. """ # agent-platform-t-compact-wire-format (#85): compact separators # rather than indent=2. content[0].text is a byte-for-byte duplicate @@ -1110,7 +1107,7 @@ def _build_content_blocks( ), ) ] - for row in rows: + for row in structured.get("results", []): cid = row["chunk_id"] blocks.append( ResourceLink( diff --git a/tests/test_search_filter.py b/tests/test_search_filter.py index 1191bc08..5c56b4eb 100644 --- a/tests/test_search_filter.py +++ b/tests/test_search_filter.py @@ -1318,3 +1318,31 @@ def test_paper_id_and_source_kind_compose(self) -> None: {"paper_id": "2401.00001", "source_kind": "textbook"}, ) assert out == [] + + +class TestResourceLinksFollowTheCap: + def test_links_only_for_rows_the_cap_kept(self, fake_resources) -> None: + """Each result row gets one resource_link. When the byte cap drops + trailing rows, their links must go too, or the client is pointed at + chunks that are not in the payload it was handed.""" + from mcp.types import ResourceLink + + from server.handlers.search import handle_search_papers + + resources = fake_resources["resources"] + # The full payload is ~3.8 KB, ~7.7 KB with the wire factor of 2, + # so this keeps roughly half the rows. + resources.config.result_byte_cap = 4096 + resources.chunks_table.set_rows([ + {"chunk_id": f"arxiv:2604.{n:05d}:abc", "paper_id": f"2604.{n:05d}", + "body_text": "x" * 4000, "_distance": 0.1 + n / 100} + for n in range(10) + ]) + + result = _run(handle_search_papers(query="Bridgeland stability", k=10)) + + kept = [r["chunk_id"] for r in result.structuredContent["results"]] + links = [b.name for b in result.content if isinstance(b, ResourceLink)] + assert result.structuredContent.get("body_truncated") is True + assert 0 < len(kept) < 10 + assert links == kept From 1b5f67fca39171755fab9a094d1005767306b995 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 16:07:36 +0000 Subject: [PATCH 2/3] feat(server): serve the chunk links the tools emit search_papers emits a resource_link per result, and get_chunk emits one when a body is over the byte cap, all to arxmcp://chunks/. No resource was registered at that URI, so every link failed with "Unknown resource". The get_chunk tool description tells agents to follow the link for the full payload. Register an arxmcp://chunks/{chunk_id} template. It returns the same object a scalar get_chunk does, with the same sanitized, delimiter- wrapped body, but without the per-response byte cap: that cap is what produced the link, so applying it again would make the link useless. Ingest bounds a chunk body to a few thousand tokens, so the uncapped read stays small. A malformed id is an error; an id not in the corpus returns found:false. Making the advertised contract true was preferred over dropping the links, because rewording the tool description would move EXPECTED_TOOL_SCHEMA_SHA256. Resources are a separate JSON-RPC method, so tools/list is unchanged, and test_mcp_resources and test_server_tool_schema pass. New tests drive get_chunk over a small cap, then read the resource_link_uri it returns and get the whole body back. On the unfixed server that read raises "Unknown resource". Closes #328. --- server/handlers/chunk.py | 22 ++++++++++ server/mcp_resources.py | 35 +++++++++++++++- tests/test_chunk_resource.py | 78 ++++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 tests/test_chunk_resource.py diff --git a/server/handlers/chunk.py b/server/handlers/chunk.py index dbd545b7..4030a959 100644 --- a/server/handlers/chunk.py +++ b/server/handlers/chunk.py @@ -28,6 +28,7 @@ from __future__ import annotations +import sys from typing import Annotated, Any from pydantic import Field @@ -139,6 +140,27 @@ async def handle_get_chunk( return envelope(structured) +async def read_chunk_in_full(chunk_id: str) -> dict[str, Any]: + """One chunk with its whole body, for the ``arxmcp://chunks/{id}`` + resource that ``get_chunk``'s over-cap ``resource_link`` points at. + + Same shape and trust treatment as a scalar ``get_chunk`` (sanitized, + delimiter-wrapped body), but without the per-response byte cap: that + cap is what produced the link, so applying it again would make the + link useless. Ingest bounds a chunk body to a few thousand tokens, so + the uncapped read stays small. + """ + if not is_valid_chunk_id(chunk_id): + raise ValueError( + f"chunk_id {chunk_id!r} does not match arxiv::<16-hex> format" + ) + r = get_resources() + row = _fetch_chunk_rows(r, [chunk_id]).get(chunk_id) + return envelope( + _build_chunk_result(chunk_id, row, False, r, byte_cap=sys.maxsize) + ) + + def _fetch_chunk_rows(r, ids: list[str]) -> dict[str, Any]: # noqa: ANN001 """One LanceDB lookup for all requested ids -> {chunk_id: row}. diff --git a/server/mcp_resources.py b/server/mcp_resources.py index e72e7f55..8c692f4f 100644 --- a/server/mcp_resources.py +++ b/server/mcp_resources.py @@ -6,7 +6,7 @@ ``tools/list`` and never enter the frozen tool-schema / BP1 prompt-cache prefix (proven byte-identical by ``notebook-surface-expansion-spike-1``). -Five resources are registered (see :func:`register_resources`): +Six resources are registered (see :func:`register_resources`): - ``arxmcp://notebooks`` — a **concrete** index resource; ``resources/read`` returns ``{count, notebooks: [{slug, display_name, uri}, …]}`` (enumeration). @@ -14,6 +14,8 @@ notebook's METADATA (NO chunk content, NO LanceDB query). - ``arxmcp://corpus-manifest`` — a **concrete** resource, generated on-read (source-truth-m3); the content-addressed corpus provenance manifest. +- ``arxmcp://chunks/{chunk_id}`` — a **template**; one chunk in full, the + target of the tool handlers' ``resource_link`` blocks. - ``arxmcp://formal/{notebook}`` and ``arxmcp://formal/{notebook}/{key}`` — **templates** over the pinned formalization a notebook serves (contract-v1). @@ -49,7 +51,13 @@ from server import corpus_manifest from server.corpus import read_corpus_version -from server.tools import ResourcesNotReadyError, get_resources, wrap_retrieved_text +from server.handlers.chunk import read_chunk_in_full +from server.tools import ( + CHUNK_RESOURCE_URI_SCHEME, + ResourcesNotReadyError, + get_resources, + wrap_retrieved_text, +) from tools._notebook_common import ( NotebookError, notebook_dir, @@ -71,6 +79,9 @@ #: source-truth-m3: a THIRD concrete resource (not a template) — the #: content-addressed corpus provenance manifest, generated on-read. CORPUS_MANIFEST_URI = "arxmcp://corpus-manifest" +#: One chunk, in full. The tool handlers emit links to it: one per +#: search_papers result, and one for any get_chunk body over the byte cap. +CHUNK_TEMPLATE_URI = f"{CHUNK_RESOURCE_URI_SCHEME}{{chunk_id}}" #: contract-v1 (derived-alg-geo-lean #175): the pinned formalization a notebook #: serves, and one record out of it. TEMPLATES on ``{notebook}``, which is what #: makes a SECOND topic repo cost zero arXMCP code. @@ -621,6 +632,25 @@ async def _corpus_manifest() -> str: ) return _wrap_json(payload, kind="manifest") + @mcp_server.resource( + CHUNK_TEMPLATE_URI, + name="chunk", + description=( + "One chunk by chunk_id, with its whole body: the same object a " + "scalar get_chunk returns, without get_chunk's per-response byte " + "cap. This is what the resource_link blocks from search_papers " + "and an over-cap get_chunk point at. A well-formed id that is not " + "in the corpus returns found:false; a malformed one is an error." + ), + mime_type="text/plain", + ) + async def _chunk(chunk_id: str) -> str: + # The body inside is already sanitized and delimiter-wrapped by the + # get_chunk builder, so the payload is serialized as is. + return json.dumps( + await read_chunk_in_full(chunk_id), sort_keys=True, ensure_ascii=False + ) + @mcp_server.resource( FORMAL_INDEX_TEMPLATE_URI, name="formal", @@ -665,6 +695,7 @@ async def _formal_record_resource(notebook: str, key: str) -> str: __all__ = [ + "CHUNK_TEMPLATE_URI", "CORPUS_MANIFEST_URI", "FORMAL_INDEX_TEMPLATE_URI", "FORMAL_RECORD_TEMPLATE_URI", diff --git a/tests/test_chunk_resource.py b/tests/test_chunk_resource.py new file mode 100644 index 00000000..941b1c16 --- /dev/null +++ b/tests/test_chunk_resource.py @@ -0,0 +1,78 @@ +"""``arxmcp://chunks/{chunk_id}``: the resource the tool handlers link to.""" + +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace + +import lancedb +import pyarrow as pa +import pytest +from mcp.server.fastmcp import FastMCP + +from ingest.schema import CHUNKS_SCHEMA_V1 +from server.handlers.chunk import handle_get_chunk +from server.mcp_resources import CHUNK_TEMPLATE_URI, register_resources +from server.tools import reset_resources_for_tests, set_resources + +CHUNK_ID = "arxiv:2401.00001:0123456789abcdef" +LONG_BODY = "Let X be a smooth projective variety. " * 500 # ~19 KB + + +@pytest.fixture +def mcp(tmp_path): + row = {field.name: None for field in CHUNKS_SCHEMA_V1} + row.update( + chunk_id=CHUNK_ID, paper_id="2401.00001", kind="theorem", + section_path=["1"], body_text=LONG_BODY, body_tokens="x", + chunker_version="v", embedder_version="v", license="", + ) + table = lancedb.connect(str(tmp_path / "lancedb")).create_table( + "chunks", data=pa.Table.from_pylist([row], schema=CHUNKS_SCHEMA_V1) + ) + set_resources(SimpleNamespace( + chunks_table=table, + corpus_info=SimpleNamespace(version=7), + # Small enough that get_chunk must truncate this body. + config=SimpleNamespace(result_byte_cap=8 * 1024), + )) + server = FastMCP("arxmcp", json_response=True) + register_resources(server) + yield server + reset_resources_for_tests() + + +def _read(server: FastMCP, uri: str) -> dict: + (content,) = asyncio.run(server.read_resource(uri)) + return json.loads(content.content) + + +def test_the_template_is_registered(mcp): + templates = asyncio.run(mcp.list_resource_templates()) + assert CHUNK_TEMPLATE_URI in {t.uriTemplate for t in templates} + + +def test_get_chunks_over_cap_link_resolves_to_the_whole_body(mcp): + """get_chunk truncates the body and points at this resource; reading + the link must give back what was cut.""" + capped = asyncio.run(handle_get_chunk(CHUNK_ID)) + assert capped["body_truncated"] is True + assert len(capped["chunk"]["body_text"]) < len(LONG_BODY) + + full = _read(mcp, capped["resource_link_uri"]) + + assert full["found"] is True + assert full.get("body_truncated") is not True + assert LONG_BODY.strip() in full["chunk"]["body_text"] + + +def test_an_id_not_in_the_corpus_is_found_false(mcp): + missing = _read(mcp, "arxmcp://chunks/arxiv:2401.00002:0123456789abcdef") + + assert missing["found"] is False + + +def test_a_malformed_id_is_an_error(mcp): + with pytest.raises(Exception, match="does not match"): + asyncio.run(mcp.read_resource("arxmcp://chunks/not-a-chunk-id")) From 40f147c4e1635de9d262c422dbe61e12da0ad912 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 16:13:49 +0000 Subject: [PATCH 3/3] fix(server): never wipe the theorem index at startup When the theorem-names SQLite file predated the store's schema version, Resources.startup opened it with the migrating path, which drops and recreates the tables empty. The served process, which writes no corpus state, wiped the index. find_lemma_by_name then answered every lookup with no matches under retrieval_mode fuzzy_jaccard, which reads exactly like a real "no such lemma". An index that was simply empty answered the same way. TheoremNamesStore.open gains migrate (default True, so the offline indexer still rebuilds). The server opens with migrate=False: an out-of-date file raises TheoremNamesSchemaStale and is left untouched. Both that and an empty store now return None, which routes the handler to its documented in_memory_scan_fallback until `python -m ingest.index_theorem_names` rebuilds the index. The cost is a slower scan, not a wrong answer. New tests: a schema-0 file with a row stays byte-for-byte as it was and is not served; an empty store is not served; a populated one is; and a lookup over the stale file finds the lemma via the fallback. On the unfixed server the first two are served and the lookup returns fuzzy_jaccard with no matches. Closes #359. --- server/resources.py | 29 +++++++++---- server/theorem_names_store.py | 27 +++++++++++- tests/test_theorem_names.py | 77 +++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 10 deletions(-) diff --git a/server/resources.py b/server/resources.py index bab8c0f5..118408fe 100644 --- a/server/resources.py +++ b/server/resources.py @@ -2132,7 +2132,10 @@ async def _open_theorem_names_store(config: Config, *, caller: str) -> Any | Non #207; see that function's docstring for why. """ try: - from server.theorem_names_store import TheoremNamesStore # noqa: PLC0415 + from server.theorem_names_store import ( # noqa: PLC0415 + TheoremNamesSchemaStale, + TheoremNamesStore, + ) tdb_path = Path(config.theorem_names_db_path) if not tdb_path.exists(): @@ -2143,12 +2146,17 @@ async def _open_theorem_names_store(config: Config, *, caller: str) -> Any | Non tdb_path, ) return None - store = await TheoremNamesStore.open(tdb_path) - # F5 (E10_S02 critique) — log the row count so an empty store - # (a recent schema bump, or a never-completed first ingest) is - # distinguishable from a populated one. An empty store + cold - # corpus would otherwise be indistinguishable from "no named - # theorems anywhere in the corpus." + # Never migrate here: rebuilding an out-of-date file would wipe it + # from the served process, which writes no corpus state. + try: + store = await TheoremNamesStore.open(tdb_path, migrate=False) + except TheoremNamesSchemaStale as exc: + logger.warning( + "%s: %s Until then find_lemma_by_name scans the chunks table.", + caller, + exc, + ) + return None row_count = await store.row_count() logger.info( "%s: opened theorem_names SQLite store at %s (rows=%d)", @@ -2157,12 +2165,17 @@ async def _open_theorem_names_store(config: Config, *, caller: str) -> Any | Non row_count, ) if row_count == 0: + # An empty store would answer every lookup with no matches + # under a normal-looking retrieval_mode, indistinguishable from + # "no such lemma". The scan fallback says what it is. logger.warning( "%s: theorem_names store is EMPTY. Re-run " "`python -m ingest.index_theorem_names` to populate it; " - "until then find_lemma_by_name returns zero matches.", + "until then find_lemma_by_name scans the chunks table.", caller, ) + await store.close() + return None return store except Exception as exc: # noqa: BLE001 — non-critical enrichment logger.warning("%s: could not open theorem_names store: %s", caller, exc) diff --git a/server/theorem_names_store.py b/server/theorem_names_store.py index 91735b75..392100f9 100644 --- a/server/theorem_names_store.py +++ b/server/theorem_names_store.py @@ -206,6 +206,18 @@ class TheoremRow: # --------------------------------------------------------------------------- +class TheoremNamesSchemaStale(Exception): + """The file predates :data:`SCHEMA_VERSION` and was not migrated.""" + + def __init__(self, db_path: Path, found: int) -> None: + super().__init__( + f"theorem_names store at {db_path} has schema {found}; this " + f"server reads {SCHEMA_VERSION}. Rebuild it with " + "`python -m ingest.index_theorem_names`." + ) + self.found = found + + class TheoremNamesStore: """SQLite-backed store for the theorem-names index. @@ -221,8 +233,15 @@ def __init__(self, db_path: Path, connection: sqlite3.Connection) -> None: self._lock = asyncio.Lock() @classmethod - async def open(cls, db_path: Path) -> TheoremNamesStore: - """Open (or create) the SQLite file at ``db_path``.""" + async def open(cls, db_path: Path, *, migrate: bool = True) -> TheoremNamesStore: + """Open (or create) the SQLite file at ``db_path``. + + A file whose schema is older than :data:`SCHEMA_VERSION` is rebuilt + empty, which only the offline indexer should do. The served process + passes ``migrate=False``: it gets :class:`TheoremNamesSchemaStale` + instead, leaves the file untouched, and falls back to scanning the + chunks table until the indexer has rebuilt it. + """ db_path.parent.mkdir(parents=True, exist_ok=True) def _open_sync() -> sqlite3.Connection: @@ -235,6 +254,9 @@ def _open_sync() -> sqlite3.Connection: conn.execute("PRAGMA synchronous=NORMAL") current = conn.execute("PRAGMA user_version").fetchone()[0] + if current < SCHEMA_VERSION and not migrate: + conn.close() + raise TheoremNamesSchemaStale(db_path, current) if current < SCHEMA_VERSION: # F5 (E10_S02 critique) — schema bump silently drops # the entire table. Log at WARNING so the operator @@ -575,6 +597,7 @@ def _row_from_sql(row: tuple) -> TheoremRow: __all__ = [ "DEFAULT_JACCARD_THRESHOLD", "SCHEMA_VERSION", + "TheoremNamesSchemaStale", "TheoremNamesStore", "TheoremRow", "dedup_key", diff --git a/tests/test_theorem_names.py b/tests/test_theorem_names.py index b28c9d07..687bd7bc 100644 --- a/tests/test_theorem_names.py +++ b/tests/test_theorem_names.py @@ -851,3 +851,80 @@ def test_all_punctuation_name_returns_empty_with_distinct_mode( assert result["retrieval_mode"] == "empty_after_normalization" finally: _run(store.close()) + + +class TestServedProcessNeverRebuildsTheStore: + """The served process must not migrate an out-of-date store (that + rebuild wipes it), and must not serve an empty store as if it were a + real "no matches"; both fall back to scanning the chunks table.""" + + @staticmethod + def _open(path: Path): + from types import SimpleNamespace + + from server.resources import _open_theorem_names_store + + return _run(_open_theorem_names_store( + SimpleNamespace(theorem_names_db_path=path), caller="test" + )) + + def test_stale_store_is_left_alone_and_not_served(self, tmp_path): + path = tmp_path / "theorem_names.db" + conn = sqlite3.connect(str(path)) + conn.execute("CREATE TABLE theorem_names (display_name TEXT)") + conn.execute("INSERT INTO theorem_names VALUES ('Yoneda Lemma')") + conn.commit() + conn.close() + + assert self._open(path) is None + + conn = sqlite3.connect(str(path)) + try: + assert conn.execute("PRAGMA user_version").fetchone()[0] == 0 + rows = conn.execute("SELECT display_name FROM theorem_names").fetchall() + finally: + conn.close() + assert rows == [("Yoneda Lemma",)] + + def test_empty_store_is_not_served(self, tmp_path): + path = tmp_path / "theorem_names.db" + _run(_run(TheoremNamesStore.open(path)).close()) + + assert self._open(path) is None + + def test_populated_store_is_served(self, tmp_path): + path = tmp_path / "theorem_names.db" + + async def _seed(): + store = await TheoremNamesStore.open(path) + await store.upsert_rows([TheoremRow( + dedup_key="k", paper_id="2401.00001", + chunk_id="arxiv:2401.00001:0000000000000001", + theorem_name="Yoneda Lemma", display_name="Yoneda Lemma", + section_path=["1"], confidence=1.0, + )]) + await store.close() + + _run(_seed()) + store = self._open(path) + try: + assert store is not None + finally: + _run(store.close()) + + def test_lookup_over_a_stale_store_still_finds_the_lemma(self, tmp_path): + path = tmp_path / "theorem_names.db" + sqlite3.connect(str(path)).close() # schema 0: stale + ct = _chunks_table_from_records(tmp_path, [{ + "chunk_id": "arxiv:2401.00001:0000000000000001", + "paper_id": "2401.00001", + "theorem_name": "Yoneda Lemma", + }]) + _install_resources(ct, theorem_names_db=self._open(path), tmp_path=tmp_path) + try: + result = _run(lemma_handler_mod.handle_find_lemma_by_name(name="Yoneda")) + finally: + server_tools._RESOURCES = None + + assert result["retrieval_mode"] == "in_memory_scan_fallback" + assert len(result["matches"]) == 1