diff --git a/server/handlers/chunk.py b/server/handlers/chunk.py index dbd545b..4030a95 100644 --- a/server/handlers/chunk.py +++ b/server/handlers/chunk.py @@ -28,6 +28,7 @@ from __future__ import annotations +import sys from typing import Annotated, Any from pydantic import Field @@ -139,6 +140,27 @@ async def handle_get_chunk( return envelope(structured) +async def read_chunk_in_full(chunk_id: str) -> dict[str, Any]: + """One chunk with its whole body, for the ``arxmcp://chunks/{id}`` + resource that ``get_chunk``'s over-cap ``resource_link`` points at. + + Same shape and trust treatment as a scalar ``get_chunk`` (sanitized, + delimiter-wrapped body), but without the per-response byte cap: that + cap is what produced the link, so applying it again would make the + link useless. Ingest bounds a chunk body to a few thousand tokens, so + the uncapped read stays small. + """ + if not is_valid_chunk_id(chunk_id): + raise ValueError( + f"chunk_id {chunk_id!r} does not match arxiv::<16-hex> format" + ) + r = get_resources() + row = _fetch_chunk_rows(r, [chunk_id]).get(chunk_id) + return envelope( + _build_chunk_result(chunk_id, row, False, r, byte_cap=sys.maxsize) + ) + + def _fetch_chunk_rows(r, ids: list[str]) -> dict[str, Any]: # noqa: ANN001 """One LanceDB lookup for all requested ids -> {chunk_id: row}. diff --git a/server/handlers/search.py b/server/handlers/search.py index c3ac803..058401a 100644 --- a/server/handlers/search.py +++ b/server/handlers/search.py @@ -716,7 +716,7 @@ async def handle_search_papers( getattr(r, "paper_metadata_store", None), rows ) structured = _cap(structured) - content = _build_content_blocks(structured, rows) + content = _build_content_blocks(structured) return CallToolResult(content=content, structuredContent=structured) # Encode query (singleflight + semaphore — two-tier concurrency). @@ -781,7 +781,7 @@ async def handle_search_papers( getattr(r, "paper_metadata_store", None), rows ) structured = _cap(structured) - content = _build_content_blocks(structured, rows) + content = _build_content_blocks(structured) return CallToolResult(content=content, structuredContent=structured) # Dense ANN over embedding_stmt only. embedding_proof is for @@ -955,7 +955,7 @@ async def handle_search_papers( # one ResourceLink per result row. E13_S04b: cap the structured # payload (defensive; closes Threat 4 partial-coverage gap G1). structured = _cap(structured) - content = _build_content_blocks(structured, rows) + content = _build_content_blocks(structured) return CallToolResult(content=content, structuredContent=structured) @@ -1071,9 +1071,7 @@ def _cap(structured: dict[str, Any]) -> dict[str, Any]: return structured -def _build_content_blocks( - structured: dict[str, Any], rows: list[dict[str, Any]] -) -> list[Any]: +def _build_content_blocks(structured: dict[str, Any]) -> list[Any]: """Build the ``content`` array per the E06_S04 contract. Block 0 is a TextContent carrying ``json.dumps(structured, @@ -1081,17 +1079,16 @@ def _build_content_blocks( dict-handler emits, so clients that only read ``content[0].text`` see the full payload. - Blocks 1..N are ResourceLink blocks in the same order as - ``rows`` (which is already sorted ``(score_desc, - chunk_id_asc)``). Each link's URI is - ``arxmcp://chunks/`` per the design note's resource - URI scheme. - - Note: ``search_papers`` does NOT call ``enforce_byte_cap`` - (only ``get_chunk`` does at v1). When a future milestone - wires cap enforcement here, the per-row ResourceLink overhead - must be added to the wire-overhead factor calibration in - ``server/tools.py``. + Blocks 1..N are ResourceLink blocks, one per row of + ``structured["results"]`` in the same order (already sorted + ``(score_desc, chunk_id_asc)``). Each link's URI is + ``arxmcp://chunks/``, served by the chunk resource in + :mod:`server.mcp_resources`. + + The links are built from ``structured`` AFTER the byte cap, so a + row the cap dropped gets no link. The links themselves are not in + the cap's measurement: about 200 bytes each as JSON, so ~10 KB for + the 50-row maximum against a 256 KB cap. """ # agent-platform-t-compact-wire-format (#85): compact separators # rather than indent=2. content[0].text is a byte-for-byte duplicate @@ -1110,7 +1107,7 @@ def _build_content_blocks( ), ) ] - for row in rows: + for row in structured.get("results", []): cid = row["chunk_id"] blocks.append( ResourceLink( diff --git a/server/mcp_resources.py b/server/mcp_resources.py index 79eb6fd..5500913 100644 --- a/server/mcp_resources.py +++ b/server/mcp_resources.py @@ -6,7 +6,7 @@ ``tools/list`` and never enter the frozen tool-schema / BP1 prompt-cache prefix (proven byte-identical by ``notebook-surface-expansion-spike-1``). -Three resources are registered (see :func:`register_resources`): +Four resources are registered (see :func:`register_resources`): - ``arxmcp://notebooks`` — a **concrete** index resource; ``resources/read`` returns ``{count, notebooks: [{slug, display_name, uri}, …]}`` (enumeration). @@ -14,6 +14,8 @@ notebook's METADATA (NO chunk content, NO LanceDB query). - ``arxmcp://corpus-manifest`` — a **concrete** resource, generated on-read (source-truth-m3); the content-addressed corpus provenance manifest. +- ``arxmcp://chunks/{chunk_id}`` — a **template**; one chunk in full, the + target of the tool handlers' ``resource_link`` blocks. This surface is **read-only** — notebook mutation stays on the ``/ui/api`` REST surface (the MCP notebook-mutation-tools path is on the project's Won't list). @@ -45,7 +47,13 @@ from typing import TYPE_CHECKING, Any from server import corpus_manifest -from server.tools import ResourcesNotReadyError, get_resources, wrap_retrieved_text +from server.handlers.chunk import read_chunk_in_full +from server.tools import ( + CHUNK_RESOURCE_URI_SCHEME, + ResourcesNotReadyError, + get_resources, + wrap_retrieved_text, +) from tools._notebook_common import ( NotebookError, notebook_dir, @@ -66,6 +74,9 @@ #: source-truth-m3: a THIRD concrete resource (not a template) — the #: content-addressed corpus provenance manifest, generated on-read. CORPUS_MANIFEST_URI = "arxmcp://corpus-manifest" +#: One chunk, in full. The tool handlers emit links to it: one per +#: search_papers result, and one for any get_chunk body over the byte cap. +CHUNK_TEMPLATE_URI = f"{CHUNK_RESOURCE_URI_SCHEME}{{chunk_id}}" #: Module-level live store (set by the lifespan via :func:`set_notebooks_store`, #: mirroring ``server.tools.set_resources``). ``None`` until startup wires it. _notebooks_store: NotebooksStore | None = None @@ -257,8 +268,28 @@ async def _corpus_manifest() -> str: ) return _wrap_json(payload, kind="manifest") + @mcp_server.resource( + CHUNK_TEMPLATE_URI, + name="chunk", + description=( + "One chunk by chunk_id, with its whole body: the same object a " + "scalar get_chunk returns, without get_chunk's per-response byte " + "cap. This is what the resource_link blocks from search_papers " + "and an over-cap get_chunk point at. A well-formed id that is not " + "in the corpus returns found:false; a malformed one is an error." + ), + mime_type="text/plain", + ) + async def _chunk(chunk_id: str) -> str: + # The body inside is already sanitized and delimiter-wrapped by the + # get_chunk builder, so the payload is serialized as is. + return json.dumps( + await read_chunk_in_full(chunk_id), sort_keys=True, ensure_ascii=False + ) + __all__ = [ + "CHUNK_TEMPLATE_URI", "CORPUS_MANIFEST_URI", "NOTEBOOKS_INDEX_URI", "NOTEBOOK_TEMPLATE_URI", diff --git a/server/resources.py b/server/resources.py index 7a9c740..455e532 100644 --- a/server/resources.py +++ b/server/resources.py @@ -2129,7 +2129,10 @@ async def _open_theorem_names_store(config: Config, *, caller: str) -> Any | Non #207; see that function's docstring for why. """ try: - from server.theorem_names_store import TheoremNamesStore # noqa: PLC0415 + from server.theorem_names_store import ( # noqa: PLC0415 + TheoremNamesSchemaStale, + TheoremNamesStore, + ) tdb_path = Path(config.theorem_names_db_path) if not tdb_path.exists(): @@ -2140,12 +2143,17 @@ async def _open_theorem_names_store(config: Config, *, caller: str) -> Any | Non tdb_path, ) return None - store = await TheoremNamesStore.open(tdb_path) - # F5 (E10_S02 critique) — log the row count so an empty store - # (a recent schema bump, or a never-completed first ingest) is - # distinguishable from a populated one. An empty store + cold - # corpus would otherwise be indistinguishable from "no named - # theorems anywhere in the corpus." + # Never migrate here: rebuilding an out-of-date file would wipe it + # from the served process, which writes no corpus state. + try: + store = await TheoremNamesStore.open(tdb_path, migrate=False) + except TheoremNamesSchemaStale as exc: + logger.warning( + "%s: %s Until then find_lemma_by_name scans the chunks table.", + caller, + exc, + ) + return None row_count = await store.row_count() logger.info( "%s: opened theorem_names SQLite store at %s (rows=%d)", @@ -2154,12 +2162,17 @@ async def _open_theorem_names_store(config: Config, *, caller: str) -> Any | Non row_count, ) if row_count == 0: + # An empty store would answer every lookup with no matches + # under a normal-looking retrieval_mode, indistinguishable from + # "no such lemma". The scan fallback says what it is. logger.warning( "%s: theorem_names store is EMPTY. Re-run " "`python -m ingest.index_theorem_names` to populate it; " - "until then find_lemma_by_name returns zero matches.", + "until then find_lemma_by_name scans the chunks table.", caller, ) + await store.close() + return None return store except Exception as exc: # noqa: BLE001 — non-critical enrichment logger.warning("%s: could not open theorem_names store: %s", caller, exc) diff --git a/server/theorem_names_store.py b/server/theorem_names_store.py index 91735b7..392100f 100644 --- a/server/theorem_names_store.py +++ b/server/theorem_names_store.py @@ -206,6 +206,18 @@ class TheoremRow: # --------------------------------------------------------------------------- +class TheoremNamesSchemaStale(Exception): + """The file predates :data:`SCHEMA_VERSION` and was not migrated.""" + + def __init__(self, db_path: Path, found: int) -> None: + super().__init__( + f"theorem_names store at {db_path} has schema {found}; this " + f"server reads {SCHEMA_VERSION}. Rebuild it with " + "`python -m ingest.index_theorem_names`." + ) + self.found = found + + class TheoremNamesStore: """SQLite-backed store for the theorem-names index. @@ -221,8 +233,15 @@ def __init__(self, db_path: Path, connection: sqlite3.Connection) -> None: self._lock = asyncio.Lock() @classmethod - async def open(cls, db_path: Path) -> TheoremNamesStore: - """Open (or create) the SQLite file at ``db_path``.""" + async def open(cls, db_path: Path, *, migrate: bool = True) -> TheoremNamesStore: + """Open (or create) the SQLite file at ``db_path``. + + A file whose schema is older than :data:`SCHEMA_VERSION` is rebuilt + empty, which only the offline indexer should do. The served process + passes ``migrate=False``: it gets :class:`TheoremNamesSchemaStale` + instead, leaves the file untouched, and falls back to scanning the + chunks table until the indexer has rebuilt it. + """ db_path.parent.mkdir(parents=True, exist_ok=True) def _open_sync() -> sqlite3.Connection: @@ -235,6 +254,9 @@ def _open_sync() -> sqlite3.Connection: conn.execute("PRAGMA synchronous=NORMAL") current = conn.execute("PRAGMA user_version").fetchone()[0] + if current < SCHEMA_VERSION and not migrate: + conn.close() + raise TheoremNamesSchemaStale(db_path, current) if current < SCHEMA_VERSION: # F5 (E10_S02 critique) — schema bump silently drops # the entire table. Log at WARNING so the operator @@ -575,6 +597,7 @@ def _row_from_sql(row: tuple) -> TheoremRow: __all__ = [ "DEFAULT_JACCARD_THRESHOLD", "SCHEMA_VERSION", + "TheoremNamesSchemaStale", "TheoremNamesStore", "TheoremRow", "dedup_key", diff --git a/tests/test_chunk_resource.py b/tests/test_chunk_resource.py new file mode 100644 index 0000000..941b1c1 --- /dev/null +++ b/tests/test_chunk_resource.py @@ -0,0 +1,78 @@ +"""``arxmcp://chunks/{chunk_id}``: the resource the tool handlers link to.""" + +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace + +import lancedb +import pyarrow as pa +import pytest +from mcp.server.fastmcp import FastMCP + +from ingest.schema import CHUNKS_SCHEMA_V1 +from server.handlers.chunk import handle_get_chunk +from server.mcp_resources import CHUNK_TEMPLATE_URI, register_resources +from server.tools import reset_resources_for_tests, set_resources + +CHUNK_ID = "arxiv:2401.00001:0123456789abcdef" +LONG_BODY = "Let X be a smooth projective variety. " * 500 # ~19 KB + + +@pytest.fixture +def mcp(tmp_path): + row = {field.name: None for field in CHUNKS_SCHEMA_V1} + row.update( + chunk_id=CHUNK_ID, paper_id="2401.00001", kind="theorem", + section_path=["1"], body_text=LONG_BODY, body_tokens="x", + chunker_version="v", embedder_version="v", license="", + ) + table = lancedb.connect(str(tmp_path / "lancedb")).create_table( + "chunks", data=pa.Table.from_pylist([row], schema=CHUNKS_SCHEMA_V1) + ) + set_resources(SimpleNamespace( + chunks_table=table, + corpus_info=SimpleNamespace(version=7), + # Small enough that get_chunk must truncate this body. + config=SimpleNamespace(result_byte_cap=8 * 1024), + )) + server = FastMCP("arxmcp", json_response=True) + register_resources(server) + yield server + reset_resources_for_tests() + + +def _read(server: FastMCP, uri: str) -> dict: + (content,) = asyncio.run(server.read_resource(uri)) + return json.loads(content.content) + + +def test_the_template_is_registered(mcp): + templates = asyncio.run(mcp.list_resource_templates()) + assert CHUNK_TEMPLATE_URI in {t.uriTemplate for t in templates} + + +def test_get_chunks_over_cap_link_resolves_to_the_whole_body(mcp): + """get_chunk truncates the body and points at this resource; reading + the link must give back what was cut.""" + capped = asyncio.run(handle_get_chunk(CHUNK_ID)) + assert capped["body_truncated"] is True + assert len(capped["chunk"]["body_text"]) < len(LONG_BODY) + + full = _read(mcp, capped["resource_link_uri"]) + + assert full["found"] is True + assert full.get("body_truncated") is not True + assert LONG_BODY.strip() in full["chunk"]["body_text"] + + +def test_an_id_not_in_the_corpus_is_found_false(mcp): + missing = _read(mcp, "arxmcp://chunks/arxiv:2401.00002:0123456789abcdef") + + assert missing["found"] is False + + +def test_a_malformed_id_is_an_error(mcp): + with pytest.raises(Exception, match="does not match"): + asyncio.run(mcp.read_resource("arxmcp://chunks/not-a-chunk-id")) diff --git a/tests/test_search_filter.py b/tests/test_search_filter.py index 1191bc0..5c56b4e 100644 --- a/tests/test_search_filter.py +++ b/tests/test_search_filter.py @@ -1318,3 +1318,31 @@ def test_paper_id_and_source_kind_compose(self) -> None: {"paper_id": "2401.00001", "source_kind": "textbook"}, ) assert out == [] + + +class TestResourceLinksFollowTheCap: + def test_links_only_for_rows_the_cap_kept(self, fake_resources) -> None: + """Each result row gets one resource_link. When the byte cap drops + trailing rows, their links must go too, or the client is pointed at + chunks that are not in the payload it was handed.""" + from mcp.types import ResourceLink + + from server.handlers.search import handle_search_papers + + resources = fake_resources["resources"] + # The full payload is ~3.8 KB, ~7.7 KB with the wire factor of 2, + # so this keeps roughly half the rows. + resources.config.result_byte_cap = 4096 + resources.chunks_table.set_rows([ + {"chunk_id": f"arxiv:2604.{n:05d}:abc", "paper_id": f"2604.{n:05d}", + "body_text": "x" * 4000, "_distance": 0.1 + n / 100} + for n in range(10) + ]) + + result = _run(handle_search_papers(query="Bridgeland stability", k=10)) + + kept = [r["chunk_id"] for r in result.structuredContent["results"]] + links = [b.name for b in result.content if isinstance(b, ResourceLink)] + assert result.structuredContent.get("body_truncated") is True + assert 0 < len(kept) < 10 + assert links == kept diff --git a/tests/test_theorem_names.py b/tests/test_theorem_names.py index b28c9d0..687bd7b 100644 --- a/tests/test_theorem_names.py +++ b/tests/test_theorem_names.py @@ -851,3 +851,80 @@ def test_all_punctuation_name_returns_empty_with_distinct_mode( assert result["retrieval_mode"] == "empty_after_normalization" finally: _run(store.close()) + + +class TestServedProcessNeverRebuildsTheStore: + """The served process must not migrate an out-of-date store (that + rebuild wipes it), and must not serve an empty store as if it were a + real "no matches"; both fall back to scanning the chunks table.""" + + @staticmethod + def _open(path: Path): + from types import SimpleNamespace + + from server.resources import _open_theorem_names_store + + return _run(_open_theorem_names_store( + SimpleNamespace(theorem_names_db_path=path), caller="test" + )) + + def test_stale_store_is_left_alone_and_not_served(self, tmp_path): + path = tmp_path / "theorem_names.db" + conn = sqlite3.connect(str(path)) + conn.execute("CREATE TABLE theorem_names (display_name TEXT)") + conn.execute("INSERT INTO theorem_names VALUES ('Yoneda Lemma')") + conn.commit() + conn.close() + + assert self._open(path) is None + + conn = sqlite3.connect(str(path)) + try: + assert conn.execute("PRAGMA user_version").fetchone()[0] == 0 + rows = conn.execute("SELECT display_name FROM theorem_names").fetchall() + finally: + conn.close() + assert rows == [("Yoneda Lemma",)] + + def test_empty_store_is_not_served(self, tmp_path): + path = tmp_path / "theorem_names.db" + _run(_run(TheoremNamesStore.open(path)).close()) + + assert self._open(path) is None + + def test_populated_store_is_served(self, tmp_path): + path = tmp_path / "theorem_names.db" + + async def _seed(): + store = await TheoremNamesStore.open(path) + await store.upsert_rows([TheoremRow( + dedup_key="k", paper_id="2401.00001", + chunk_id="arxiv:2401.00001:0000000000000001", + theorem_name="Yoneda Lemma", display_name="Yoneda Lemma", + section_path=["1"], confidence=1.0, + )]) + await store.close() + + _run(_seed()) + store = self._open(path) + try: + assert store is not None + finally: + _run(store.close()) + + def test_lookup_over_a_stale_store_still_finds_the_lemma(self, tmp_path): + path = tmp_path / "theorem_names.db" + sqlite3.connect(str(path)).close() # schema 0: stale + ct = _chunks_table_from_records(tmp_path, [{ + "chunk_id": "arxiv:2401.00001:0000000000000001", + "paper_id": "2401.00001", + "theorem_name": "Yoneda Lemma", + }]) + _install_resources(ct, theorem_names_db=self._open(path), tmp_path=tmp_path) + try: + result = _run(lemma_handler_mod.handle_find_lemma_by_name(name="Yoneda")) + finally: + server_tools._RESOURCES = None + + assert result["retrieval_mode"] == "in_memory_scan_fallback" + assert len(result["matches"]) == 1