Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions server/handlers/chunk.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@

from __future__ import annotations

import sys
from typing import Annotated, Any

from pydantic import Field
Expand Down Expand Up @@ -139,6 +140,27 @@ async def handle_get_chunk(
return envelope(structured)


async def read_chunk_in_full(chunk_id: str) -> dict[str, Any]:
"""One chunk with its whole body, for the ``arxmcp://chunks/{id}``
resource that ``get_chunk``'s over-cap ``resource_link`` points at.

Same shape and trust treatment as a scalar ``get_chunk`` (sanitized,
delimiter-wrapped body), but without the per-response byte cap: that
cap is what produced the link, so applying it again would make the
link useless. Ingest bounds a chunk body to a few thousand tokens, so
the uncapped read stays small.
"""
if not is_valid_chunk_id(chunk_id):
raise ValueError(
f"chunk_id {chunk_id!r} does not match arxiv:<paper_id>:<16-hex> format"
)
r = get_resources()
row = _fetch_chunk_rows(r, [chunk_id]).get(chunk_id)
return envelope(
_build_chunk_result(chunk_id, row, False, r, byte_cap=sys.maxsize)
)


def _fetch_chunk_rows(r, ids: list[str]) -> dict[str, Any]: # noqa: ANN001
"""One LanceDB lookup for all requested ids -> {chunk_id: row}.

Expand Down
33 changes: 15 additions & 18 deletions server/handlers/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -716,7 +716,7 @@ async def handle_search_papers(
getattr(r, "paper_metadata_store", None), rows
)
structured = _cap(structured)
content = _build_content_blocks(structured, rows)
content = _build_content_blocks(structured)
return CallToolResult(content=content, structuredContent=structured)

# Encode query (singleflight + semaphore — two-tier concurrency).
Expand Down Expand Up @@ -781,7 +781,7 @@ async def handle_search_papers(
getattr(r, "paper_metadata_store", None), rows
)
structured = _cap(structured)
content = _build_content_blocks(structured, rows)
content = _build_content_blocks(structured)
return CallToolResult(content=content, structuredContent=structured)

# Dense ANN over embedding_stmt only. embedding_proof is for
Expand Down Expand Up @@ -955,7 +955,7 @@ async def handle_search_papers(
# one ResourceLink per result row. E13_S04b: cap the structured
# payload (defensive; closes Threat 4 partial-coverage gap G1).
structured = _cap(structured)
content = _build_content_blocks(structured, rows)
content = _build_content_blocks(structured)
return CallToolResult(content=content, structuredContent=structured)


Expand Down Expand Up @@ -1071,27 +1071,24 @@ def _cap(structured: dict[str, Any]) -> dict[str, Any]:
return structured


def _build_content_blocks(
structured: dict[str, Any], rows: list[dict[str, Any]]
) -> list[Any]:
def _build_content_blocks(structured: dict[str, Any]) -> list[Any]:
"""Build the ``content`` array per the E06_S04 contract.

Block 0 is a TextContent carrying ``json.dumps(structured,
indent=2, sort_keys=True)`` — same shape FastMCP's default
dict-handler emits, so clients that only read
``content[0].text`` see the full payload.

Blocks 1..N are ResourceLink blocks in the same order as
``rows`` (which is already sorted ``(score_desc,
chunk_id_asc)``). Each link's URI is
``arxmcp://chunks/<chunk_id>`` per the design note's resource
URI scheme.

Note: ``search_papers`` does NOT call ``enforce_byte_cap``
(only ``get_chunk`` does at v1). When a future milestone
wires cap enforcement here, the per-row ResourceLink overhead
must be added to the wire-overhead factor calibration in
``server/tools.py``.
Blocks 1..N are ResourceLink blocks, one per row of
``structured["results"]`` in the same order (already sorted
``(score_desc, chunk_id_asc)``). Each link's URI is
``arxmcp://chunks/<chunk_id>``, served by the chunk resource in
:mod:`server.mcp_resources`.

The links are built from ``structured`` AFTER the byte cap, so a
row the cap dropped gets no link. The links themselves are not in
the cap's measurement: about 200 bytes each as JSON, so ~10 KB for
the 50-row maximum against a 256 KB cap.
"""
# agent-platform-t-compact-wire-format (#85): compact separators
# rather than indent=2. content[0].text is a byte-for-byte duplicate
Expand All @@ -1110,7 +1107,7 @@ def _build_content_blocks(
),
)
]
for row in rows:
for row in structured.get("results", []):
cid = row["chunk_id"]
blocks.append(
ResourceLink(
Expand Down
35 changes: 33 additions & 2 deletions server/mcp_resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,16 @@
``tools/list`` and never enter the frozen tool-schema / BP1 prompt-cache prefix
(proven byte-identical by ``notebook-surface-expansion-spike-1``).

Three resources are registered (see :func:`register_resources`):
Four resources are registered (see :func:`register_resources`):

- ``arxmcp://notebooks`` — a **concrete** index resource; ``resources/read``
returns ``{count, notebooks: [{slug, display_name, uri}, …]}`` (enumeration).
- ``arxmcp://notebooks/{slug}`` — a **template**; ``resources/read`` returns ONE
notebook's METADATA (NO chunk content, NO LanceDB query).
- ``arxmcp://corpus-manifest`` — a **concrete** resource, generated on-read
(source-truth-m3); the content-addressed corpus provenance manifest.
- ``arxmcp://chunks/{chunk_id}`` — a **template**; one chunk in full, the
target of the tool handlers' ``resource_link`` blocks.

This surface is **read-only** — notebook mutation stays on the ``/ui/api`` REST
surface (the MCP notebook-mutation-tools path is on the project's Won't list).
Expand Down Expand Up @@ -45,7 +47,13 @@
from typing import TYPE_CHECKING, Any

from server import corpus_manifest
from server.tools import ResourcesNotReadyError, get_resources, wrap_retrieved_text
from server.handlers.chunk import read_chunk_in_full
from server.tools import (
CHUNK_RESOURCE_URI_SCHEME,
ResourcesNotReadyError,
get_resources,
wrap_retrieved_text,
)
from tools._notebook_common import (
NotebookError,
notebook_dir,
Expand All @@ -66,6 +74,9 @@
#: source-truth-m3: a THIRD concrete resource (not a template) — the
#: content-addressed corpus provenance manifest, generated on-read.
CORPUS_MANIFEST_URI = "arxmcp://corpus-manifest"
#: One chunk, in full. The tool handlers emit links to it: one per
#: search_papers result, and one for any get_chunk body over the byte cap.
CHUNK_TEMPLATE_URI = f"{CHUNK_RESOURCE_URI_SCHEME}{{chunk_id}}"
#: Module-level live store (set by the lifespan via :func:`set_notebooks_store`,
#: mirroring ``server.tools.set_resources``). ``None`` until startup wires it.
_notebooks_store: NotebooksStore | None = None
Expand Down Expand Up @@ -257,8 +268,28 @@ async def _corpus_manifest() -> str:
)
return _wrap_json(payload, kind="manifest")

@mcp_server.resource(
CHUNK_TEMPLATE_URI,
name="chunk",
description=(
"One chunk by chunk_id, with its whole body: the same object a "
"scalar get_chunk returns, without get_chunk's per-response byte "
"cap. This is what the resource_link blocks from search_papers "
"and an over-cap get_chunk point at. A well-formed id that is not "
"in the corpus returns found:false; a malformed one is an error."
),
mime_type="text/plain",
)
async def _chunk(chunk_id: str) -> str:
# The body inside is already sanitized and delimiter-wrapped by the
# get_chunk builder, so the payload is serialized as is.
return json.dumps(
await read_chunk_in_full(chunk_id), sort_keys=True, ensure_ascii=False
)


__all__ = [
"CHUNK_TEMPLATE_URI",
"CORPUS_MANIFEST_URI",
"NOTEBOOKS_INDEX_URI",
"NOTEBOOK_TEMPLATE_URI",
Expand Down
29 changes: 21 additions & 8 deletions server/resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -2129,7 +2129,10 @@ async def _open_theorem_names_store(config: Config, *, caller: str) -> Any | Non
#207; see that function's docstring for why.
"""
try:
from server.theorem_names_store import TheoremNamesStore # noqa: PLC0415
from server.theorem_names_store import ( # noqa: PLC0415
TheoremNamesSchemaStale,
TheoremNamesStore,
)

tdb_path = Path(config.theorem_names_db_path)
if not tdb_path.exists():
Expand All @@ -2140,12 +2143,17 @@ async def _open_theorem_names_store(config: Config, *, caller: str) -> Any | Non
tdb_path,
)
return None
store = await TheoremNamesStore.open(tdb_path)
# F5 (E10_S02 critique) — log the row count so an empty store
# (a recent schema bump, or a never-completed first ingest) is
# distinguishable from a populated one. An empty store + cold
# corpus would otherwise be indistinguishable from "no named
# theorems anywhere in the corpus."
# Never migrate here: rebuilding an out-of-date file would wipe it
# from the served process, which writes no corpus state.
try:
store = await TheoremNamesStore.open(tdb_path, migrate=False)
except TheoremNamesSchemaStale as exc:
logger.warning(
"%s: %s Until then find_lemma_by_name scans the chunks table.",
caller,
exc,
)
return None
row_count = await store.row_count()
logger.info(
"%s: opened theorem_names SQLite store at %s (rows=%d)",
Expand All @@ -2154,12 +2162,17 @@ async def _open_theorem_names_store(config: Config, *, caller: str) -> Any | Non
row_count,
)
if row_count == 0:
# An empty store would answer every lookup with no matches
# under a normal-looking retrieval_mode, indistinguishable from
# "no such lemma". The scan fallback says what it is.
logger.warning(
"%s: theorem_names store is EMPTY. Re-run "
"`python -m ingest.index_theorem_names` to populate it; "
"until then find_lemma_by_name returns zero matches.",
"until then find_lemma_by_name scans the chunks table.",
caller,
)
await store.close()
return None
return store
except Exception as exc: # noqa: BLE001 — non-critical enrichment
logger.warning("%s: could not open theorem_names store: %s", caller, exc)
Expand Down
27 changes: 25 additions & 2 deletions server/theorem_names_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,18 @@ class TheoremRow:
# ---------------------------------------------------------------------------


class TheoremNamesSchemaStale(Exception):
"""The file predates :data:`SCHEMA_VERSION` and was not migrated."""

def __init__(self, db_path: Path, found: int) -> None:
super().__init__(
f"theorem_names store at {db_path} has schema {found}; this "
f"server reads {SCHEMA_VERSION}. Rebuild it with "
"`python -m ingest.index_theorem_names`."
)
self.found = found


class TheoremNamesStore:
"""SQLite-backed store for the theorem-names index.

Expand All @@ -221,8 +233,15 @@ def __init__(self, db_path: Path, connection: sqlite3.Connection) -> None:
self._lock = asyncio.Lock()

@classmethod
async def open(cls, db_path: Path) -> TheoremNamesStore:
"""Open (or create) the SQLite file at ``db_path``."""
async def open(cls, db_path: Path, *, migrate: bool = True) -> TheoremNamesStore:
"""Open (or create) the SQLite file at ``db_path``.

A file whose schema is older than :data:`SCHEMA_VERSION` is rebuilt
empty, which only the offline indexer should do. The served process
passes ``migrate=False``: it gets :class:`TheoremNamesSchemaStale`
instead, leaves the file untouched, and falls back to scanning the
chunks table until the indexer has rebuilt it.
"""
db_path.parent.mkdir(parents=True, exist_ok=True)

def _open_sync() -> sqlite3.Connection:
Expand All @@ -235,6 +254,9 @@ def _open_sync() -> sqlite3.Connection:
conn.execute("PRAGMA synchronous=NORMAL")

current = conn.execute("PRAGMA user_version").fetchone()[0]
if current < SCHEMA_VERSION and not migrate:
conn.close()
raise TheoremNamesSchemaStale(db_path, current)
if current < SCHEMA_VERSION:
# F5 (E10_S02 critique) — schema bump silently drops
# the entire table. Log at WARNING so the operator
Expand Down Expand Up @@ -575,6 +597,7 @@ def _row_from_sql(row: tuple) -> TheoremRow:
__all__ = [
"DEFAULT_JACCARD_THRESHOLD",
"SCHEMA_VERSION",
"TheoremNamesSchemaStale",
"TheoremNamesStore",
"TheoremRow",
"dedup_key",
Expand Down
78 changes: 78 additions & 0 deletions tests/test_chunk_resource.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
"""``arxmcp://chunks/{chunk_id}``: the resource the tool handlers link to."""

from __future__ import annotations

import asyncio
import json
from types import SimpleNamespace

import lancedb
import pyarrow as pa
import pytest
from mcp.server.fastmcp import FastMCP

from ingest.schema import CHUNKS_SCHEMA_V1
from server.handlers.chunk import handle_get_chunk
from server.mcp_resources import CHUNK_TEMPLATE_URI, register_resources
from server.tools import reset_resources_for_tests, set_resources

CHUNK_ID = "arxiv:2401.00001:0123456789abcdef"
LONG_BODY = "Let X be a smooth projective variety. " * 500 # ~19 KB


@pytest.fixture
def mcp(tmp_path):
row = {field.name: None for field in CHUNKS_SCHEMA_V1}
row.update(
chunk_id=CHUNK_ID, paper_id="2401.00001", kind="theorem",
section_path=["1"], body_text=LONG_BODY, body_tokens="x",
chunker_version="v", embedder_version="v", license="",
)
table = lancedb.connect(str(tmp_path / "lancedb")).create_table(
"chunks", data=pa.Table.from_pylist([row], schema=CHUNKS_SCHEMA_V1)
)
set_resources(SimpleNamespace(
chunks_table=table,
corpus_info=SimpleNamespace(version=7),
# Small enough that get_chunk must truncate this body.
config=SimpleNamespace(result_byte_cap=8 * 1024),
))
server = FastMCP("arxmcp", json_response=True)
register_resources(server)
yield server
reset_resources_for_tests()


def _read(server: FastMCP, uri: str) -> dict:
(content,) = asyncio.run(server.read_resource(uri))
return json.loads(content.content)


def test_the_template_is_registered(mcp):
templates = asyncio.run(mcp.list_resource_templates())
assert CHUNK_TEMPLATE_URI in {t.uriTemplate for t in templates}


def test_get_chunks_over_cap_link_resolves_to_the_whole_body(mcp):
"""get_chunk truncates the body and points at this resource; reading
the link must give back what was cut."""
capped = asyncio.run(handle_get_chunk(CHUNK_ID))
assert capped["body_truncated"] is True
assert len(capped["chunk"]["body_text"]) < len(LONG_BODY)

full = _read(mcp, capped["resource_link_uri"])

assert full["found"] is True
assert full.get("body_truncated") is not True
assert LONG_BODY.strip() in full["chunk"]["body_text"]


def test_an_id_not_in_the_corpus_is_found_false(mcp):
missing = _read(mcp, "arxmcp://chunks/arxiv:2401.00002:0123456789abcdef")

assert missing["found"] is False


def test_a_malformed_id_is_an_error(mcp):
with pytest.raises(Exception, match="does not match"):
asyncio.run(mcp.read_resource("arxmcp://chunks/not-a-chunk-id"))
Loading
Loading