From 83b57fca05313d7944d9634cc20d1aefbca6b217 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 00:46:52 +0000 Subject: [PATCH 1/8] fix(server): trim over-cap result lists in linear time cap_result_list re-serialized the whole payload after each row it dropped, so trimming n rows cost bytes quadratic in n, on the event loop, exactly when the payload was largest. A 200-row payload trimmed to nothing serialized ~6.4M characters for a 61K-character response. Serialize each row once and subtract its bytes as it is dropped. With the default separators a nested value serializes to the same bytes it occupies inside its parent, so the size stays exact: a row costs its own bytes plus the ", " before it. New tests compare the rows kept against a brute-force reference over random payloads with non-ASCII text and escapes, and bound the bytes serialized to a small multiple of the payload. The docstring drops milestone and finding ids. Closes #351. --- server/tools.py | 31 +++++----- tests/test_cap_result_list.py | 105 ++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 13 deletions(-) create mode 100644 tests/test_cap_result_list.py diff --git a/server/tools.py b/server/tools.py index c76d0de..e47975d 100644 --- a/server/tools.py +++ b/server/tools.py @@ -872,7 +872,7 @@ def cap_result_list( list_key: str, chunk_id: str | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: - """E13_S04b — cap the byte size of a multi-result tool envelope. + """Cap the byte size of a multi-result tool envelope. The companion to :func:`enforce_byte_cap` for handlers whose over-cap surface is the aggregate response (a list of rows in @@ -882,14 +882,10 @@ def cap_result_list( :func:`enforce_byte_cap` signature so callers can treat both helpers uniformly. - Closes the F1 finding from the E13_S04b adversary critique: - multi-result tools previously called ``enforce_byte_cap(payload)`` - with the default ``body_text_path=("body_text",)``, which silently - no-ops on payloads with no top-level ``body_text`` key. - ``body_truncated=True`` was set but the payload bytes were - unchanged — a 500 KB response stayed 500 KB on the wire. + :func:`enforce_byte_cap` cannot do this job: it trims a top-level + ``body_text`` and silently no-ops on payloads without one. - This helper iteratively pops the trailing row from + This helper drops trailing rows from ``structured_content[list_key]`` until the serialized payload (with the :data:`_WIRE_OVERHEAD_FACTOR` wire-overhead correction) fits under @@ -920,20 +916,29 @@ def cap_result_list( """ cap = get_resources().config.result_byte_cap - def _measure(d: dict[str, Any]) -> int: + def _nbytes(value: Any) -> int: return len( - json.dumps(d, ensure_ascii=False, sort_keys=True).encode("utf-8") - ) * _WIRE_OVERHEAD_FACTOR + json.dumps(value, ensure_ascii=False, sort_keys=True).encode("utf-8") + ) - if _measure(structured_content) <= cap: + size = _nbytes(structured_content) + if size * _WIRE_OVERHEAD_FACTOR <= cap: return structured_content, [] # Over cap — deep-copy so we never mutate the caller's payload. truncated = json.loads(json.dumps(structured_content)) rows = truncated.get(list_key) if isinstance(rows, list): - while rows and _measure(truncated) > cap: + # Serialize each row once and subtract as rows go, rather than + # re-serializing the whole payload per dropped row. With the + # default separators a nested value serializes to the same bytes + # it occupies inside its parent, so the arithmetic is exact: a + # row costs its own bytes plus the ", " before it, and the last + # remaining row has no separator. + row_sizes = [_nbytes(row) for row in rows] + while rows and size * _WIRE_OVERHEAD_FACTOR > cap: rows.pop() + size -= row_sizes.pop() + (2 if rows else 0) truncated["body_truncated"] = True blocks: list[dict[str, Any]] = [] if chunk_id is not None: diff --git a/tests/test_cap_result_list.py b/tests/test_cap_result_list.py new file mode 100644 index 0000000..f22f639 --- /dev/null +++ b/tests/test_cap_result_list.py @@ -0,0 +1,105 @@ +"""``cap_result_list`` drops the lowest-ranked rows until the payload fits.""" + +from __future__ import annotations + +import json +import random +from types import SimpleNamespace + +import pytest + +from server import tools +from server.tools import cap_result_list, reset_resources_for_tests, set_resources + + +def _wire_size(payload: dict) -> int: + text = json.dumps(payload, ensure_ascii=False, sort_keys=True) + return len(text.encode("utf-8")) * tools._WIRE_OVERHEAD_FACTOR + + +def _rows_that_fit(payload: dict, cap: int) -> int: + """Reference answer: the most leading rows whose payload fits.""" + rows = payload["results"] + for n in range(len(rows), -1, -1): + if _wire_size({**payload, "results": rows[:n]}) <= cap: + return n + return 0 + + +@pytest.fixture +def byte_cap(): + def install(cap: int) -> None: + set_resources(SimpleNamespace(config=SimpleNamespace(result_byte_cap=cap))) + + yield install + reset_resources_for_tests() + + +def _payload(seed: int, n_rows: int) -> dict: + rng = random.Random(seed) + alphabet = "abcxyz ∂∑ℤ→é\"\\" + rows = [ + { + "chunk_id": f"2401.{i:05d}#c{i}", + "score": rng.random(), + "snippet": "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 300))), + "meta": {"kind": rng.choice(["theorem", "proof"]), "n": [i, None, True]}, + } + for i in range(n_rows) + ] + return {"query": "zeta ∑", "results": rows, "retrieval_mode": "hybrid"} + + +@pytest.mark.parametrize("seed", range(5)) +def test_keeps_exactly_the_rows_that_fit(byte_cap, seed): + payload = _payload(seed, 40) + full = _wire_size(payload) + for cap in sorted({full // 7, full // 3, full // 2, full - 2, 50, 1}): + byte_cap(cap) + out, blocks = cap_result_list(payload, "results") + + assert len(out["results"]) == _rows_that_fit(payload, cap) + assert out["results"] == payload["results"][: len(out["results"])] + assert out["body_truncated"] is True + assert blocks == [] + + +def test_under_cap_payload_is_returned_untouched(byte_cap): + payload = _payload(0, 5) + byte_cap(_wire_size(payload)) + + out, _ = cap_result_list(payload, "results") + + assert out is payload + + +def test_caller_payload_is_not_mutated(byte_cap): + payload = _payload(1, 20) + before = json.dumps(payload, sort_keys=True) + byte_cap(_wire_size(payload) // 4) + + cap_result_list(payload, "results") + + assert json.dumps(payload, sort_keys=True) == before + + +def test_serialization_work_is_linear_in_payload_size(byte_cap, monkeypatch): + """Trimming must not re-serialize the whole payload per dropped row, + which costs bytes quadratic in the row count.""" + payload = _payload(2, 200) + payload_chars = len(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + byte_cap(200) + serialized_chars = 0 + real_dumps = json.dumps + + def counting_dumps(*args, **kwargs): + nonlocal serialized_chars + text = real_dumps(*args, **kwargs) + serialized_chars += len(text) + return text + + monkeypatch.setattr(tools.json, "dumps", counting_dumps) + out, _ = cap_result_list(payload, "results") + + assert out["results"] == [] + assert serialized_chars <= 5 * payload_chars From 758ab9fe8409d2aab8cee1916ff82fb7f030d006 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 00:47:48 +0000 Subject: [PATCH 2/8] fix(server): never leave a finished parse running The parse worker's success-path status write sat outside its try/except. If it raised (SQLite "database is locked" under a concurrent status poll is the likely case), the task ended exceptionally, the row stayed parse_status=running, and every later upload to that notebook was refused until a restart. The restart's orphan sweep then recorded a parse that had succeeded as failed. Retry the complete write twice with a short backoff. If it still fails, write a terminal failed status that says the parse finished but could not be recorded, so the notebook is unblocked. The failure path already guarded its own write; the success path now matches. Closes #367. --- server/parse_tracker.py | 67 ++++++++++++++++++++++++++++------- tests/test_parse_tracker.py | 70 +++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 12 deletions(-) diff --git a/server/parse_tracker.py b/server/parse_tracker.py index b800580..f5b8b37 100644 --- a/server/parse_tracker.py +++ b/server/parse_tracker.py @@ -26,7 +26,9 @@ see ``pending`` indefinitely. - **``done_callback`` removes the tracker entry** but the DB row update is INSIDE the worker function so terminal-state writes - happen even if the callback is delayed. + happen even if the callback is delayed. Every path out of the + worker attempts a terminal write: a row left ``running`` blocks + every later upload to that notebook until a restart. - **Lifespan shutdown cancels in-flight tasks** then a startup-time ``mark_orphaned_parses_failed()`` sweep cleans up any leftover ``running`` rows from a hard crash (synthesis FM-4). @@ -96,6 +98,11 @@ def _redact_path_prefix(text: str) -> str: #: an absolute-path prefix (leading ``/`` or ``\``) up to and #: including ``var/arxmcp/`` and replaces it with the bare relative #: form. Non-greedy so only the prefix is consumed. +#: Seconds to wait before each retry of the success-path status write. +#: A failure there is most likely SQLite ``database is locked`` under a +#: concurrent status poll, which clears within moments. +_COMPLETE_WRITE_RETRY_DELAYS_S: tuple[float, ...] = (0.25, 1.0) + _ABS_PATH_PREFIX_RE = re.compile(r"[/\\][\w /\\.\-]*?var[/\\]arxmcp[/\\]") @@ -206,6 +213,29 @@ def _on_task_done(self, slug: str, task: asyncio.Task) -> None: "parse task for slug=%s raised: %s", slug, exc, ) + @staticmethod + async def _record_complete( + store: NotebooksStore, slug: str, html_path_str: str, + ) -> None: + """Write ``complete``, retrying a transient failure.""" + for delay in (*_COMPLETE_WRITE_RETRY_DELAYS_S, None): + try: + await store.update_parse_status( + slug, + store.PARSE_STATUS_COMPLETE, + parse_error="", + parsed_html_path=html_path_str, + ) + return + except Exception: + if delay is None: + raise + logger.warning( + "recording parse complete for slug=%s failed; " + "retrying in %.2fs", slug, delay, exc_info=True, + ) + await asyncio.sleep(delay) + async def _run_parse( self, *, @@ -279,17 +309,30 @@ async def _run_parse( return # Success: record the parsed_html_path scrubbed to its - # ``var/arxmcp/``-relative form (m6 F1). Storing the - # absolute path would leak the operator's home dir - # through the /parse-status JSON — the m9 redact_paths - # discipline applies here too. - html_path_str = redact_html_path(render_result.output_html_path) - await store.update_parse_status( - slug, - store.PARSE_STATUS_COMPLETE, - parse_error="", - parsed_html_path=html_path_str, - ) + # ``var/arxmcp/``-relative form. Storing the absolute path + # would leak the operator's home dir through the + # /parse-status JSON. + try: + html_path_str = redact_html_path(render_result.output_html_path) + await self._record_complete(store, slug, html_path_str) + except Exception as e: # noqa: BLE001 + # The HTML is on disk, but a row left ``running`` would + # block every later upload to this notebook, so settle + # for a terminal ``failed`` that says what happened. + logger.exception( + "parse for slug=%s finished but its status could not " + "be recorded", slug, + ) + with contextlib.suppress(Exception): + await store.update_parse_status( + slug, + store.PARSE_STATUS_FAILED, + parse_error=_format_parse_error( + "parse finished but recording it failed: " + f"{type(e).__name__}: {e}" + ), + ) + return logger.info( "parse complete: slug=%s html=%s errors=%d " "mineru_wall=%.1fs render_wall=%.2fs", diff --git a/tests/test_parse_tracker.py b/tests/test_parse_tracker.py index 15d3b0e..1c2dcf2 100644 --- a/tests/test_parse_tracker.py +++ b/tests/test_parse_tracker.py @@ -14,6 +14,7 @@ import asyncio import dataclasses import os +import sqlite3 import subprocess from pathlib import Path from unittest.mock import AsyncMock, MagicMock @@ -288,6 +289,75 @@ def test_dataclass_render_result_is_frozen(self) -> None: # --------------------------------------------------------------------------- +class TestSuccessPathStatusWrite: + """A finished parse must never leave its row ``running``.""" + + @staticmethod + def _run_successful_parse( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, store: MagicMock, + ) -> None: + from ingest import textbook_parser, textbook_renderer + from server import parse_tracker + + mineru_result = _make_mineru_result(tmp_path) + render_result = _make_render_result(tmp_path) + monkeypatch.setattr( + textbook_parser, "run_mineru_sandboxed", + lambda pdf_path, output_dir: mineru_result, + ) + monkeypatch.setattr( + textbook_renderer, "render_mineru_to_html", + lambda r, p, pid: render_result, + ) + monkeypatch.setattr( + parse_tracker, "_COMPLETE_WRITE_RETRY_DELAYS_S", (0, 0), raising=False, + ) + + async def _run() -> None: + tracker = ParseTaskTracker() + await tracker.start_parse( + slug="test-slug", + pdf_path=tmp_path / "x.pdf", + paper_id="textbook:x", + output_dir=tmp_path / "out", + parsed_dir=tmp_path / "parsed", + store=store, + ) + + asyncio.run(_run()) + + def test_transient_write_failure_still_records_complete( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + store = _make_store_mock() + store.update_parse_status = AsyncMock( + side_effect=[sqlite3.OperationalError("database is locked"), True], + ) + + self._run_successful_parse(tmp_path, monkeypatch, store) + + statuses = [c.args[1] for c in store.update_parse_status.call_args_list] + assert statuses == ["complete", "complete"] + + def test_persistent_write_failure_ends_failed_not_running( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + store = _make_store_mock() + + async def update(slug, status, **kwargs): + if status == "complete": + raise sqlite3.OperationalError("database is locked") + return True + + store.update_parse_status = AsyncMock(side_effect=update) + + self._run_successful_parse(tmp_path, monkeypatch, store) + + last = store.update_parse_status.call_args + assert last.args[1] == "failed" + assert "recording it failed" in last.kwargs["parse_error"] + + def _pid_alive(pid: int) -> bool: """Direct signal-0 probe. Deliberately not the code under test.""" try: From 6b446ee654028ef03a2096acc473c05a3bd2c5a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 00:48:16 +0000 Subject: [PATCH 3/8] fix(tests): anchor the equations-parity gate to the repo The gate looked for the parsed corpus at a working-directory-relative path. Started from anywhere but the repo root (a subdirectory, an IDE runner), it found nothing and skipped with "parsed corpus absent or too small (0 equations)" while the corpus was present, so the math-fidelity gate silently did not run. Resolve the path from the test file, as every other test module does, and name the full path in the skip message. The module's docstrings drop milestone ids. Closes #375. --- tests/eval/test_equations_parity.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/eval/test_equations_parity.py b/tests/eval/test_equations_parity.py index 3c7e7b9..0ee91a9 100644 --- a/tests/eval/test_equations_parity.py +++ b/tests/eval/test_equations_parity.py @@ -1,6 +1,6 @@ -"""retrieval-unlocks-m4 — the equations-fixture parity gate. +"""The equations-fixture parity gate. -m4's acceptance is conditional: the TED route is reported for a LaTeX +The LaTeX route's acceptance is conditional: the TED route is reported for a LaTeX query *"when the equations-fixture parity check has passed"*. This is that check, kept as a regression guard so a future ``latex2mathml`` bump (or a change to ``parse_mathml_to_tree``) cannot silently degrade the @@ -60,7 +60,9 @@ pytestmark = pytest.mark.eval -_CORPUS = Path("var/arxmcp/corpus/parsed") +# Anchored to the repo root, not the working directory, so the gate runs +# (or skips) the same way whichever directory pytest is started from. +_CORPUS = Path(__file__).resolve().parents[2] / "var" / "arxmcp" / "corpus" / "parsed" #: Gate. Below this, the LaTeX route is not safe to leave enabled and #: ``Config.eq_latex_route`` should default False (or the pooled @@ -108,8 +110,7 @@ def corpus_pairs(): if len(pairs) < _POOL: pytest.skip( f"parsed corpus absent or too small ({len(pairs)} equations; " - f"need {_POOL}). Populate var/arxmcp/corpus/parsed/ to run " - f"the m4 parity gate." + f"need {_POOL}). Populate {_CORPUS} to run the parity gate." ) random.Random(7).shuffle(pairs) return pairs @@ -210,8 +211,8 @@ def test_canonicalization_erases_parallel_markup(self): Without this, every stored tree carries a Content-MathML annotation subtree the query side never has, and parity is - impossible regardless of converter quality — this was the m4 - blocker, and it also silently degraded the already-shipped + impossible regardless of converter quality — this was the LaTeX + route's blocker, and it also silently degraded the already-shipped MathML-input route. """ latexml = ( @@ -247,13 +248,13 @@ def test_font_variants_collapse_KNOWN_LIMITATION(self): are indistinguishable. This is a DELIBERATE, documented loss, not a regression: the stored corpus already collapses them, because LaTeXML encodes the variant in a ``mathvariant`` attribute that - parse_mathml_to_tree has dropped since E10_S03. NFKC only makes + parse_mathml_to_tree has always dropped. NFKC only makes the query side match that pre-existing collapse. This test PINS the known behaviour so a future change that (accidentally or deliberately) starts distinguishing variants is a conscious decision with a failing test, not a silent surprise. See ``_fold_text`` for the full rationale and the path to fixing it - (preserve mathvariant on both sides — an E10_S03-scope re-index).""" + (preserve mathvariant on both sides — a full re-index).""" blackboard = "ℂ" # ℂ (U+2102) script = "𝒞" # 𝒞 (U+1D49E) plain = "C" From 358a3a9483b2463c6bc45b5b45222ae83e0c4ca1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 23:57:48 +0000 Subject: [PATCH 4/8] fix(ops): let the restore drill check the citation graph The restore drill could never validate the citation graph, for two stacked reasons: - It looked only for a DIRECTORY named index/kuzu, but the pinned kuzu stores a database as a single file there. A restored graph was always reported absent, which the drill accepts, so the check passed without looking at the graph. - Once found, it queried MATCH (p:Paper). The schema's node table is papers, so the query raised and the drill would have called a healthy graph unreadable. Accept a file or a directory directly under index/, and count the papers table. A new test builds a graph with the production schema at the path ingest writes it, inserts two papers, and expects a count of 2; the unfixed drill returns None. Docstrings drop finding ids. Refs #302: the other half, checkpointing every WAL database before backup and not just notebooks.db, is not in this change. --- ops/restore_drill_check.py | 28 +++++++++++----------------- tests/test_restore_drill.py | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/ops/restore_drill_check.py b/ops/restore_drill_check.py index 327fd3a..c1c1375 100644 --- a/ops/restore_drill_check.py +++ b/ops/restore_drill_check.py @@ -60,18 +60,14 @@ def _locate_lancedb_root(restore_path: Path) -> Path | None: def _locate_kuzu_root(restore_path: Path) -> Path | None: """Find the restored Kùzu graph by searching for the canonical ``var/arxmcp/index/kuzu`` segment under - ``restore_path``. Returns ``None`` if no Kùzu directory - was captured. - - Closes adversary F1 (Kùzu half). + ``restore_path``. Returns ``None`` if no graph was captured. """ - # Kùzu doesn't have a single canonical marker file, so we - # look for the directory name. Filter to directories whose - # immediate parent is the expected ``index/`` so we don't - # match arbitrary "kuzu" directories the operator may have - # added under the snapshot. + # The pinned Kùzu stores a database as a single FILE at that path; + # older releases used a directory. Accept either, but only directly + # under ``index/``, so an unrelated "kuzu" entry elsewhere in the + # snapshot is not mistaken for the graph. for entry in restore_path.rglob("kuzu"): - if entry.is_dir() and entry.parent.name == "index": + if entry.parent.name == "index" and (entry.is_file() or entry.is_dir()): return entry return None @@ -118,12 +114,9 @@ def smoke_check_lancedb(restore_path: Path) -> int: def smoke_check_kuzu(restore_path: Path) -> int | None: """Open the restored Kùzu graph and return paper count. - Returns ``None`` if the Kùzu directory is absent (acceptable - — some restore paths may exclude the citation graph). Raises - ``RuntimeError`` if the directory exists but is unreadable. - - Closes adversary F1 (Kùzu half): finds the Kùzu directory - via rglob rather than hardcoding the path prefix. + Returns ``None`` if the graph is absent (acceptable — some + restore paths may exclude the citation graph). Raises + ``RuntimeError`` if it exists but is unreadable. """ kuzu_path = _locate_kuzu_root(restore_path) if kuzu_path is None: @@ -145,7 +138,8 @@ def smoke_check_kuzu(restore_path: Path) -> int | None: db = kuzu.Database(str(kuzu_path), read_only=True) conn = kuzu.Connection(db) # Restored DB may be empty; we only require it to open. - result = conn.execute("MATCH (p:Paper) RETURN count(p) AS c") + # ``papers`` is the node table ingest.kuzudb_schema creates. + result = conn.execute("MATCH (p:papers) RETURN count(p) AS c") paper_count = 0 while result.has_next(): row = result.get_next() diff --git a/tests/test_restore_drill.py b/tests/test_restore_drill.py index 4702ae0..bd7c530 100644 --- a/tests/test_restore_drill.py +++ b/tests/test_restore_drill.py @@ -214,6 +214,28 @@ def test_returns_none_when_kuzu_dir_absent(self, tmp_path): # No kuzu directory at all. assert smoke_check_kuzu(tmp_path) is None + def test_counts_papers_in_a_real_restored_graph(self, tmp_path): + """A graph built by the production schema, at the path ingest + writes it, must be found and counted.""" + kuzu = pytest.importorskip("kuzu") + from ingest.kuzudb_schema import apply_schema + + graph = tmp_path / "var" / "arxmcp" / "index" / "kuzu" + graph.parent.mkdir(parents=True) + apply_schema(graph) + db = kuzu.Database(str(graph)) + conn = kuzu.Connection(db) + try: + for paper_id in ("2401.00001", "2401.00002"): + conn.execute( + "CREATE (:papers {paper_id: $id})", {"id": paper_id} + ) + finally: + conn.close() + db.close() + + assert smoke_check_kuzu(tmp_path) == 2 + # --------------------------------------------------------------------------- # run_check end-to-end From 36776506f35c52f7a0465243753abf0a4d65d0df Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 00:14:09 +0000 Subject: [PATCH 5/8] fix(server): accept ARXMCP_TRUST_REMOTE_CODE at startup ARXMCP_TRUST_REMOTE_CODE=1 is the documented way to opt in to trust_remote_code. server.model_loader reads it straight from the environment, but the startup scan only accepts ARXMCP_* names that are Config fields, so setting it made the server refuse to start. The scan now also accepts a small set of variables the server reads directly, each name imported from the module that reads it. A Config field was the other option, but a pydantic bool would accept "true" and "yes" while model_loader deliberately accepts only "1", and the ingest CLIs, which have no Config, read the variable too. A new test builds the app with the variable set; on the unfixed code it fails with the issue's error. The scan's docstring drops milestone and finding ids. Closes #290. --- server/main.py | 42 ++++++++++++++++-------------------- server/model_loader.py | 9 +++++--- tests/test_server_startup.py | 10 +++++++++ 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/server/main.py b/server/main.py index b77be7b..8a60329 100644 --- a/server/main.py +++ b/server/main.py @@ -91,6 +91,7 @@ from server.health import ( router as health_router, ) +from server.model_loader import TRUST_REMOTE_CODE_ENV from server.resources import Resources logger = logging.getLogger(__name__) @@ -408,33 +409,28 @@ def _format_unknown_arxmcp_env_var( ) +#: ``ARXMCP_*`` variables the server process reads straight from the +#: environment rather than through ``Config``. Each name is imported from +#: the module that reads it, so the two cannot drift apart. +_DIRECT_SERVER_ENV_VARS: frozenset[str] = frozenset({TRUST_REMOTE_CODE_ENV}) + + def _scan_unknown_arxmcp_env_vars(config: Config) -> None: - """Reject any ``ARXMCP_*`` env var not declared on :class:`Config`. - - Closes F4 from the E06_S01 critique. ``pydantic-settings``'s - ``extra="forbid"`` only fires for direct ``__init__`` kwargs — - NOT for env-var input. So a typo like ``ARXMCP_BIND_HOST_TYPO`` - is silently ignored at the Pydantic layer. This scan walks - ``os.environ`` for every ``ARXMCP_*`` key and asserts it maps to - a declared field. (The original docstring also cited - ``ARXMCP_OTEL_ENDPOINT`` as an example of the "documented-but- - unimplemented" class — m1 critique F2: ``ARXMCP_OTEL_ENDPOINT`` - is now declared on ``Config`` (E14 observability landed) and - would be accepted, so the example was retired.) - - ``onboarding-uplift-m1`` replaces the prior 30-line declared-var - dump with per-var hints: an ingest-tool carve-out for - ``ARXMCP_CONTACT_EMAIL`` (the documented footgun the m1 doc sweep - removed from CLAUDE.md / README.md / Makefile), a - ``difflib.get_close_matches`` (``n=1, cutoff=0.7``) suggestion for - typos like ``ARXMCP_BIND_HOTS``, and a short-form "nearest 3" - fallback otherwise. The dynamic ``Config.model_fields`` source is - preserved — adding a new ``Config`` field automatically extends - ``declared`` (m1 synthesis FM-5 mitigation). + """Reject any ``ARXMCP_*`` env var the server does not read. + + ``pydantic-settings``'s ``extra="forbid"`` only fires for direct + ``__init__`` kwargs, not for env-var input, so a typo like + ``ARXMCP_BIND_HOST_TYPO`` would otherwise be silently ignored. This + scan walks ``os.environ`` and accepts a name only if it is a + ``Config`` field (derived from ``Config.model_fields``, so a new + field extends it automatically) or one of + :data:`_DIRECT_SERVER_ENV_VARS`. Rejected names get a hint: a + tailored one for known ingest-tool vars, a close-match suggestion + for typos, or the three nearest declared names. """ declared = frozenset( f"ARXMCP_{name.upper()}" for name in Config.model_fields - ) + ) | _DIRECT_SERVER_ENV_VARS unknown = sorted( env_name for env_name in os.environ diff --git a/server/model_loader.py b/server/model_loader.py index 64abf4f..07c84fb 100644 --- a/server/model_loader.py +++ b/server/model_loader.py @@ -93,7 +93,9 @@ def validate_model_revision(revision: str, *, model_name: str) -> None: #: refuses values other than the literal string ``"1"`` so the #: operator must opt in explicitly (no fuzzy truthiness like #: ``True`` / ``yes`` / ``on``). -_TRUST_REMOTE_CODE_ENV = "ARXMCP_TRUST_REMOTE_CODE" +#: The server process reads it here, not through ``Config``, so +#: ``server.main``'s unknown-variable scan imports this name to accept it. +TRUST_REMOTE_CODE_ENV = "ARXMCP_TRUST_REMOTE_CODE" def resolve_trust_remote_code() -> bool: @@ -107,14 +109,14 @@ def resolve_trust_remote_code() -> bool: "``trust_remote_code=False`` is the default; enabling it requires ``ARXMCP_TRUST_REMOTE_CODE=1`` and logs a WARN." """ - raw = os.environ.get(_TRUST_REMOTE_CODE_ENV, "") + raw = os.environ.get(TRUST_REMOTE_CODE_ENV, "") if raw == "1": logger.warning( "%s=1: model loaders will pass trust_remote_code=True. " "This re-enables the modeling-script RCE vector from " "Threat 6 (.claude/notes/08-security-observability-ops.md). " "Production deployments must leave this unset.", - _TRUST_REMOTE_CODE_ENV, + TRUST_REMOTE_CODE_ENV, ) return True return False @@ -194,6 +196,7 @@ def assert_no_bin_in_snapshot(model_name: str, revision: str) -> None: __all__ = [ "ModelPinningError", "assert_no_bin_in_snapshot", + "TRUST_REMOTE_CODE_ENV", "resolve_trust_remote_code", "validate_model_revision", ] diff --git a/tests/test_server_startup.py b/tests/test_server_startup.py index 061d7dd..1bddc7c 100644 --- a/tests/test_server_startup.py +++ b/tests/test_server_startup.py @@ -1318,6 +1318,16 @@ def test_create_app_rejects_unknown_env(self, monkeypatch, seeded_lancedb): with pytest.raises(ValueError, match="unknown ARXMCP_"): create_app(cfg) + def test_create_app_accepts_the_trust_remote_code_escape_hatch( + self, monkeypatch, seeded_lancedb + ): + """ARXMCP_TRUST_REMOTE_CODE is read by server.model_loader, not + Config. Setting it must not stop the server from being built.""" + monkeypatch.setenv("ARXMCP_TRUST_REMOTE_CODE", "1") + cfg = Config(lancedb_path=seeded_lancedb) + + create_app(cfg) + # --- onboarding-uplift-m1 (AC2 + AC9): close-match + carve-out ----- def test_close_match_suggestion_for_typo(self, monkeypatch): From 3c8d4da301e8d9812f4cffd937eaf11d8b2e7d88 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 00:21:55 +0000 Subject: [PATCH 6/8] fix(server): read only summary columns in get_paper get_paper pulled every column of up to 10,000 chunk rows to compute a chunk count, a section count and two version strings: both 1024-float embedding columns and the full body text included, for a response that uses none of them. Select the three columns it reads. A new test runs the handler against a real LanceDB table, checks the summary, and checks that no embedding or body column was fetched; the unfixed handler fetches all of them. Comments in the handler drop issue and milestone ids. Closes #353. --- server/handlers/paper.py | 35 ++++++------- tests/test_handlers_paper.py | 98 ++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 17 deletions(-) create mode 100644 tests/test_handlers_paper.py diff --git a/server/handlers/paper.py b/server/handlers/paper.py index 72f3086..61b449b 100644 --- a/server/handlers/paper.py +++ b/server/handlers/paper.py @@ -119,6 +119,14 @@ def _cap(payload: dict[str, Any]) -> dict[str, Any]: SUPPORTED_SOURCE_KINDS: frozenset[str] = ALL_SOURCE_KINDS +#: The chunk-row columns ``get_paper`` summarizes. +_SUMMARY_COLUMNS: tuple[str, ...] = ( + "section_path", + "chunker_version", + "embedder_version", +) + + def _unsupported_envelope( paper_id: str, exc: UnsupportedSourceKind ) -> dict[str, Any]: @@ -142,32 +150,25 @@ def _unsupported_envelope( async def handle_get_paper( paper_id: Annotated[str, Field(min_length=1, description="arXiv paper id")], ) -> dict[str, Any]: - # agent-platform-t-inert-args-cleanup (#82): the former ``version`` - # arg was accepted-and-ignored (never referenced) over a corpus that - # stores no per-version rows. Removed rather than left as a lie; an - # agent that needs a specific version parses the ``vN`` suffix it - # already carries on ``paper_id``. - # F3 fix from the E06_S03 critique: validate before using - # paper_id in a SQL-style WHERE clause. - # arXMCP#209: admit the full identifier domain that ``search_papers`` - # emits. This gated on the arXiv-only validator and raised "does not - # match the arXiv id format" at a ``textbook:`` id the server had just - # handed the caller. The rows are in the same chunks table under the - # same ``paper_id`` column, reached by the same escaped predicate - # below, so there was never a data reason to refuse them. + # There is no ``version`` argument: the corpus stores no per-version + # rows, and an agent that needs one already carries the ``vN`` suffix + # on ``paper_id``. + # Validate before paper_id reaches the WHERE clause. Every id kind + # search_papers can return is admitted: textbook rows live in the same + # chunks table under the same column. try: admit_paper_id(paper_id, SUPPORTED_SOURCE_KINDS) except UnsupportedSourceKind as exc: return _cap(_unsupported_envelope(paper_id, exc)) r = get_resources() - # Filter to rows matching paper_id. LanceDB doesn't expose a - # native group-by; pull the matching rows in Arrow and aggregate - # in Python. For the 50-paper corpus the worst case is ~100s of - # rows per paper — fast enough. + # LanceDB has no group-by, so pull the paper's rows and aggregate + # here. Only the columns read below: without a select, every row + # carries both 1024-float embedding columns and the full body text. arrow = ( r.chunks_table.search() .where(f"paper_id = '{_escape(paper_id)}'", prefilter=True) + .select(list(_SUMMARY_COLUMNS)) .limit(10000) .to_arrow() ) diff --git a/tests/test_handlers_paper.py b/tests/test_handlers_paper.py new file mode 100644 index 0000000..5aa8575 --- /dev/null +++ b/tests/test_handlers_paper.py @@ -0,0 +1,98 @@ +"""``get_paper`` summarizes a paper's chunk rows from the chunks table.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import lancedb +import pyarrow as pa +import pytest + +from ingest.schema import CHUNKS_SCHEMA_V1, EMBEDDING_COLUMN_NAMES +from server.handlers.paper import handle_get_paper +from server.tools import reset_resources_for_tests, set_resources + + +def _row(paper_id: str, n: int, section: str) -> dict: + row = {field.name: None for field in CHUNKS_SCHEMA_V1} + row.update( + chunk_id=f"{paper_id}#c{n}", + paper_id=paper_id, + kind="theorem", + section_path=[section, f"{section}.{n}"], + body_text="x" * 500, + body_tokens="x", + embedding_stmt=[0.1] * 1024, + chunker_version="chunker-v9", + embedder_version="embedder-v4", + ) + return row + + +class _RecordingTable: + """Hands queries to a real LanceDB table and keeps what they fetched.""" + + def __init__(self, table): + self._table = table + self.fetched: list[pa.Table] = [] + + def search(self, *args, **kwargs): + builder = self._table.search(*args, **kwargs) + real_to_arrow = builder.to_arrow + + def to_arrow(*a, **k): + result = real_to_arrow(*a, **k) + self.fetched.append(result) + return result + + builder.to_arrow = to_arrow + return builder + + +@pytest.fixture +def chunks_table(tmp_path): + rows = [_row("2401.00001", n, s) for n, s in enumerate(["1", "1", "2"])] + rows.append(_row("2401.00002", 0, "1")) + db = lancedb.connect(str(tmp_path / "lancedb")) + table = db.create_table( + "chunks", data=pa.Table.from_pylist(rows, schema=CHUNKS_SCHEMA_V1) + ) + recording = _RecordingTable(table) + set_resources( + SimpleNamespace( + chunks_table=recording, + corpus_info=SimpleNamespace(version=3), + config=SimpleNamespace(result_byte_cap=256 * 1024), + paper_metadata_store=None, + ) + ) + yield recording + reset_resources_for_tests() + + +def test_summarizes_the_papers_rows(chunks_table): + out = asyncio.run(handle_get_paper("2401.00001")) + + paper = out["paper"] + assert out["found"] is True + assert paper["chunk_count"] == 3 + assert paper["section_count"] == 2 + assert paper["chunker_version"] == "chunker-v9" + assert paper["embedder_version"] == "embedder-v4" + + +def test_unknown_paper_is_not_found(chunks_table): + out = asyncio.run(handle_get_paper("2401.09999")) + + assert out["found"] is False + + +def test_reads_no_embedding_or_body_columns(chunks_table): + """Each embedding is 1024 floats per row; a summary needs neither + them nor the body text.""" + asyncio.run(handle_get_paper("2401.00001")) + + (fetched,) = chunks_table.fetched + leaked = set(fetched.column_names) & {*EMBEDDING_COLUMN_NAMES, "embedding_eq", "body_text"} + assert leaked == set() From 7684e9fbe3c5e87b7a5c94a06c425289cb729ba0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 00:24:11 +0000 Subject: [PATCH 7/8] fix(ops): checkpoint every WAL database before backup The backup checkpointed only notebooks.db before snapshotting, but each notebook's documents.db and paper_metadata.db also run in WAL mode and sit inside the backed-up var/arxmcp/notebooks tree. A file-level copy taken while committed frames sit in a -wal restores stale or as "database disk image is malformed". The helper gains --backup-set , which checkpoints notebooks.db plus every *.db under notebooks/ and prints one " " line each. It skips each notebook's cache/ retrieval.db, which the backup excludes. The wrapper loops over the lines: any degraded database adds its own -wal/-shm sidecars and marks the snapshot partial, as notebooks.db alone did before. New tests build real WAL databases with committed frames in the -wal and check that every backed-up one is truncated and the query cache is left alone; the unfixed helper rejects the flag. The global theorem_names.db is not in the backup set at all; that is #360. Refs #302: with the graph half in this branch, this completes it. --- ops/checkpoint_notebooks_db.py | 34 +++++++++++++++-- ops/cron/arxmcp-backup.sh | 53 ++++++++++++++------------- tests/test_backup_wrapper.py | 3 +- tests/test_checkpoint_notebooks_db.py | 47 ++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 31 deletions(-) diff --git a/ops/checkpoint_notebooks_db.py b/ops/checkpoint_notebooks_db.py index c6dcee8..be4b0e8 100644 --- a/ops/checkpoint_notebooks_db.py +++ b/ops/checkpoint_notebooks_db.py @@ -1,6 +1,8 @@ -"""TRUNCATE-checkpoint notebooks.db WAL before a restic backup. +"""TRUNCATE-checkpoint SQLite WAL files before a restic backup. -notebook-ops-hardening-m1. +Every SQLite database in the backup set runs in WAL mode (see +:func:`backup_set_databases`). The rest of this note uses +``notebooks.db`` as the example. ``notebooks.db`` runs in WAL mode (``journal_mode=WAL`` + ``synchronous=FULL`` + ``fullfsync=ON`` from notebook-ops-hardening-m2), @@ -114,10 +116,36 @@ def checkpoint( return last +def backup_set_databases(var_root: Path) -> list[Path]: + """Every SQLite database the backup copies, under ``var_root``. + + That is ``cache/notebooks.db`` plus each notebook's stores under + ``notebooks/`` (``documents.db``, ``paper_metadata.db``). Each runs in + WAL mode, so each needs the same checkpoint. A notebook's + ``cache/retrieval.db`` is a regenerable query cache that the backup + excludes, so it is skipped here too. + """ + databases = [var_root / "cache" / "notebooks.db"] + notebooks = var_root / "notebooks" + if notebooks.is_dir(): + databases += sorted( + db + for db in notebooks.rglob("*.db") + if not (db.name == "retrieval.db" and db.parent.name == "cache") + ) + return databases + + def main(argv: list[str]) -> int: + if len(argv) == 3 and argv[1] == "--backup-set": + # One " " line per database, for the backup wrapper. + for db in backup_set_databases(Path(argv[2])): + print(f"{checkpoint(db)} {db}") + return 0 if len(argv) != 2: print( - "usage: checkpoint_notebooks_db.py ", + "usage: checkpoint_notebooks_db.py \n" + " checkpoint_notebooks_db.py --backup-set ", file=sys.stderr, ) return 64 diff --git a/ops/cron/arxmcp-backup.sh b/ops/cron/arxmcp-backup.sh index 4890d5a..f30052f 100755 --- a/ops/cron/arxmcp-backup.sh +++ b/ops/cron/arxmcp-backup.sh @@ -131,34 +131,35 @@ BACKUP_PATHS=( "'"${REPO_ROOT}"'/var/arxmcp/cache/notebooks.db" ) -# notebook-ops-hardening-m1 (CRITICAL): TRUNCATE-checkpoint the -# notebooks.db WAL before the snapshot so the file-level copy is -# self-consistent without -wal/-shm sidecars. The helper retries on a -# busy checkpoint. Only a CLEAN status (ok/absent/no-wal) means the main -# file alone is safe to back up. A residual busy/locked status (a reader -# held an open transaction through all retries) means committed frames -# remain ONLY in the un-backed-up -wal: a main-file-only copy taken then -# is stale OR malformed-on-restore (live-verified: "database disk image -# is malformed"). So on a degraded status we (a) ALSO back up the -# -wal/-shm sidecars for this run so the captured set can recover, and -# (b) force backup_status=partial so the sentinel flags it and `forget` -# retains the prior good snapshot. stderr is NOT discarded — the helper -# diagnostics must reach the cron/journal log. +# TRUNCATE-checkpoint the WAL of every SQLite database in the backup set +# (notebooks.db plus the documents.db and paper_metadata.db of every notebook) +# before the snapshot, so each file-level copy is self-consistent without +# its -wal/-shm sidecars. The helper retries a busy checkpoint and prints +# one " " line per database. Only a CLEAN status +# (ok/absent/no-wal) means the main file alone is safe to back up. A +# residual busy/locked status means committed frames remain ONLY in the +# -wal, so a main-file-only copy is stale OR malformed-on-restore +# ("database disk image is malformed"). On a degraded status we (a) ALSO +# back up the -wal/-shm sidecars of that database so the captured set can +# recover, and (b) force backup_status=partial. stderr is NOT discarded: +# the diagnostics of the helper must reach the cron/journal log. NOTEBOOKS_DB="'"${REPO_ROOT}"'/var/arxmcp/cache/notebooks.db" -CHECKPOINT_STATUS="$(python3 "'"${REPO_ROOT}"'/ops/checkpoint_notebooks_db.py" "${NOTEBOOKS_DB}" || echo error)" +CHECKPOINT_REPORT="$(python3 "'"${REPO_ROOT}"'/ops/checkpoint_notebooks_db.py" --backup-set "'"${REPO_ROOT}"'/var/arxmcp" || echo "error ${NOTEBOOKS_DB}")" CHECKPOINT_DEGRADED=0 -case "${CHECKPOINT_STATUS}" in - ok|absent|no-wal) ;; - *) - CHECKPOINT_DEGRADED=1 - echo "WARN: notebooks.db WAL checkpoint status=${CHECKPOINT_STATUS}; committed frames may remain only in the un-backed-up -wal, so the captured notebooks.db alone may be stale OR corrupt-on-restore. Backing up the -wal/-shm sidecars too and marking this backup partial." >&2 - for SIDECAR in "${NOTEBOOKS_DB}-wal" "${NOTEBOOKS_DB}-shm"; do - if [ -f "${SIDECAR}" ]; then - BACKUP_PATHS+=("${SIDECAR}") - fi - done - ;; -esac +while read -r CHECKPOINT_STATUS DB; do + case "${CHECKPOINT_STATUS}" in + ok|absent|no-wal) ;; + *) + CHECKPOINT_DEGRADED=1 + echo "WARN: ${DB} WAL checkpoint status=${CHECKPOINT_STATUS}; committed frames may remain only in the -wal, so the captured file alone may be stale OR corrupt-on-restore. Backing up its -wal/-shm sidecars too and marking this backup partial." >&2 + for SIDECAR in "${DB}-wal" "${DB}-shm"; do + if [ -f "${SIDECAR}" ]; then + BACKUP_PATHS+=("${SIDECAR}") + fi + done + ;; + esac +done <<< "${CHECKPOINT_REPORT}" # Every snapshot carries arxmcp-clean or arxmcp-partial. Retention below # counts only clean (and pre-tag, untagged) snapshots, because restic keeps diff --git a/tests/test_backup_wrapper.py b/tests/test_backup_wrapper.py index 7232ff4..67a0ef2 100644 --- a/tests/test_backup_wrapper.py +++ b/tests/test_backup_wrapper.py @@ -227,8 +227,7 @@ def test_degraded_checkpoint_marks_partial(self): assert "CHECKPOINT_DEGRADED=1" in text # The degraded path forces partial and appends the WAL sidecars. assert 'BACKUP_PATHS+=("${SIDECAR}")' in text - assert "${NOTEBOOKS_DB}-wal" in text - assert "${NOTEBOOKS_DB}-shm" in text + assert '"${DB}-wal" "${DB}-shm"' in text # And the final status is forced partial when degraded. The token # comes from the shared vocabulary (arXMCP#202), not a literal — # see ops/cron/backup-status-lib.sh and tests/ diff --git a/tests/test_checkpoint_notebooks_db.py b/tests/test_checkpoint_notebooks_db.py index 97b7f72..2f273f2 100644 --- a/tests/test_checkpoint_notebooks_db.py +++ b/tests/test_checkpoint_notebooks_db.py @@ -150,3 +150,50 @@ def test_directory_in_place_of_db_returns_locked(self, tmp_path): status = checkpoint(fake, attempts=2, delay_s=0.0) assert status == "locked" assert status not in CLEAN_STATUSES + + +class TestBackupSet: + """``--backup-set`` checkpoints every SQLite database the backup copies, + not only notebooks.db.""" + + def test_checkpoints_every_backed_up_database(self, tmp_path, capsys): + from ops.checkpoint_notebooks_db import main + + var = tmp_path / "var" / "arxmcp" + backed_up = [ + var / "cache" / "notebooks.db", + var / "notebooks" / "demo" / "documents.db", + var / "notebooks" / "demo" / "paper_metadata.db", + ] + query_cache = var / "notebooks" / "demo" / "cache" / "retrieval.db" + writers = [] + for db in [*backed_up, query_cache]: + db.parent.mkdir(parents=True, exist_ok=True) + _wal_db_with_frames(db) + # An open, idle connection keeps committed frames in the WAL + # without blocking a checkpoint. + conn = sqlite3.connect(str(db)) + conn.execute("INSERT INTO t VALUES (2)") + conn.commit() + writers.append(conn) + try: + assert main(["checkpoint", "--backup-set", str(var)]) == 0 + report = dict( + reversed(line.split(" ", 1)) + for line in capsys.readouterr().out.splitlines() + ) + + assert report == {str(db): "ok" for db in backed_up} + for db in backed_up: + assert Path(f"{db}-wal").stat().st_size == 0 + assert Path(f"{query_cache}-wal").stat().st_size > 0 + finally: + for conn in writers: + conn.close() + + def test_missing_notebooks_db_reports_absent(self, tmp_path, capsys): + from ops.checkpoint_notebooks_db import main + + var = tmp_path / "var" / "arxmcp" + assert main(["checkpoint", "--backup-set", str(var)]) == 0 + assert capsys.readouterr().out == f"absent {var / 'cache' / 'notebooks.db'}\n" From ce72c89fe5352d68cc84e6a3a6b90f107fbffab8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 00:26:21 +0000 Subject: [PATCH 8/8] fix(ingest): never reuse an earlier parse's output MinerU writes into a scratch directory that is reused for every parse of the same paper, under the same file stem. The output locator took whatever markdown it found there, so a re-parse that exited 0 without writing anything (an encrypted or undecodable PDF, say) returned the previous run's markdown, the renderer re-rendered the old book, and the parse was recorded complete. Note the wall-clock time before MinerU starts and ignore output files older than it, less a 2 s margin for coarse filesystem mtimes. A run that leaves only stale output now fails with a message saying so. New tests: over an hour-old earlier parse, a run that writes nothing raises, and a run that rewrites the same files returns the new content. Docstrings and error messages drop milestone and finding ids. Closes #368. --- ingest/textbook_parser.py | 54 +++++++++++++++++++++++++---------- tests/test_textbook_parser.py | 47 ++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 15 deletions(-) diff --git a/ingest/textbook_parser.py b/ingest/textbook_parser.py index 0cb7006..b7a365a 100644 --- a/ingest/textbook_parser.py +++ b/ingest/textbook_parser.py @@ -101,6 +101,9 @@ #: cdm_eval.py::_run_subprocess_with_pgkill. _DRAIN_TIMEOUT_S: int = 5 +#: Coarsest mtime resolution a scratch directory might sit on (FAT: 2 s). +_MTIME_GRANULARITY_S: float = 2.0 + #: Platform on which RLIMIT_AS is enforceable. Verified live test on #: Darwin 25.4.0 / Apple M4 Max: setrlimit(RLIMIT_AS, (4GB, 4GB)) raises #: ValueError because the macOS kernel keeps the hard limit at @@ -331,47 +334,65 @@ def _tail(s: str, n: int = _OUTPUT_TAIL_BYTES) -> str: return encoded[-n:].decode("utf-8", errors="replace") -def _locate_outputs(output_dir: Path, pdf_stem: str) -> tuple[Path, Path]: +def _locate_outputs( + output_dir: Path, + pdf_stem: str, + *, + written_after: float | None = None, +) -> tuple[Path, Path]: """Find the markdown + content_list.json that MinerU produced. MinerU 3.2.0 with ``-b pipeline -m auto`` produces: output_dir//auto/.md output_dir//auto/_content_list.json - Per research-synthesis §D3: probe the documented path first, - fall back to a glob, raise RuntimeError if no markdown found - (FM-4 mitigation). + Probe the documented path first, fall back to a glob, and raise + RuntimeError if no markdown is found. The fallback only accepts an + md + content_list pair that shares a parent directory, never files + from different parse subdirectories. - The fallback ONLY accepts md+content_list pairs that share a - parent directory — never pairs files from different parse - subdirectories. Closes m5 adversary F4. + ``written_after`` is a wall-clock time: files modified before it are + ignored. The scratch directory is reused for every parse of the same + paper, so without it a run that exits 0 but writes nothing would + hand back the previous run's output as its own. """ + + def fresh(*paths: Path) -> bool: + return written_after is None or all( + path.stat().st_mtime >= written_after for path in paths + ) + direct_md = output_dir / pdf_stem / "auto" / f"{pdf_stem}.md" direct_cl = ( output_dir / pdf_stem / "auto" / f"{pdf_stem}_content_list.json" ) - if direct_md.is_file() and direct_cl.is_file(): + if direct_md.is_file() and direct_cl.is_file() and fresh(direct_md, direct_cl): return direct_md, direct_cl - md_candidates = sorted(output_dir.rglob("*.md")) + all_md = sorted(output_dir.rglob("*.md")) + md_candidates = [md for md in all_md if fresh(md)] if not md_candidates: + if all_md: + raise RuntimeError( + f"MinerU exited 0 but wrote no new markdown under " + f"{output_dir!s}; the {len(all_md)} .md file(s) there are " + f"left over from an earlier parse and were not used." + ) listing = sorted(p.relative_to(output_dir) for p in output_dir.rglob("*")) raise RuntimeError( f"MinerU exited 0 but no .md file found under " f"{output_dir!s}. Directory contents: {listing!r}. The " - f"MinerU output-tree convention may have changed; see " - f"research-synthesis §D3 (FM-4)." + f"MinerU output-tree convention may have changed." ) - # Closes m5 F4: pair-match in same parent dir, do not cross-pair. for md in md_candidates: cl = md.with_name(f"{md.stem}_content_list.json") - if cl.is_file(): + if cl.is_file() and fresh(cl): return md, cl raise RuntimeError( f"MinerU exited 0 and produced markdown candidates " f"({[str(p.relative_to(output_dir)) for p in md_candidates]!r}) " f"but no paired _content_list.json in the same parent dir for " - f"any of them. Output tree partial — m6 cannot consume." + f"any of them. Output tree partial — the renderer cannot consume it." ) @@ -542,6 +563,9 @@ def run_mineru_sandboxed( ) started = time.monotonic() + # Wall clock, for comparing against output mtimes; less a margin for + # filesystems that store mtimes coarsely (FAT keeps 2 s). + written_after = time.time() - _MTIME_GRANULARITY_S proc = subprocess.Popen( cmd, cwd=str(output_dir), @@ -594,7 +618,7 @@ def run_mineru_sandboxed( ) markdown_path, content_list_path = _locate_outputs( - output_dir, pdf_path.stem, + output_dir, pdf_path.stem, written_after=written_after, ) logger.info( diff --git a/tests/test_textbook_parser.py b/tests/test_textbook_parser.py index be1f192..a0f5959 100644 --- a/tests/test_textbook_parser.py +++ b/tests/test_textbook_parser.py @@ -24,6 +24,7 @@ import signal import subprocess import sys +import time from pathlib import Path from unittest.mock import MagicMock, patch @@ -813,6 +814,52 @@ def test_default_timeout_used_when_none( assert comm_kwargs["timeout"] == sentinel_timeout +class TestReparseIgnoresEarlierOutput: + """The scratch directory is reused for every parse of a paper, so a + run must only accept output it wrote itself.""" + + @staticmethod + def _earlier_parse(tmp_path: Path, stem: str = "book") -> tuple[Path, Path]: + pdf = tmp_path / f"{stem}.pdf" + pdf.write_bytes(b"%PDF-1.4\n%fake\n") + out_dir = tmp_path / "out" + auto = out_dir / stem / "auto" + auto.mkdir(parents=True) + an_hour_ago = time.time() - 3600 + for name, text in ((f"{stem}.md", "# Old book\n"), (f"{stem}_content_list.json", "[]")): + (auto / name).write_text(text) + os.utime(auto / name, (an_hour_ago, an_hour_ago)) + return pdf, out_dir + + def test_run_that_writes_nothing_fails( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + pdf, out_dir = self._earlier_parse(tmp_path) + monkeypatch.setenv("ARXMCP_MINERU_BIN", _create_fake_bin(tmp_path)) + with patch.object(subprocess, "Popen", return_value=_fake_proc()), \ + pytest.raises(RuntimeError, match="earlier parse"): + run_mineru_sandboxed(pdf, out_dir, timeout_s=60) + + def test_run_that_rewrites_the_output_returns_it( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + pdf, out_dir = self._earlier_parse(tmp_path) + monkeypatch.setenv("ARXMCP_MINERU_BIN", _create_fake_bin(tmp_path)) + proc = _fake_proc() + + def mineru_writes(*args, **kwargs): + auto = out_dir / "book" / "auto" + (auto / "book.md").write_text("# New book\n") + (auto / "book_content_list.json").write_text("[]") + return ("", "") + + proc.communicate.side_effect = mineru_writes + with patch.object(subprocess, "Popen", return_value=proc): + result = run_mineru_sandboxed(pdf, out_dir, timeout_s=60) + + assert result.markdown_path.read_text() == "# New book\n" + + def _create_fake_bin(tmp_path: Path) -> str: bin_path = tmp_path / "fake_mineru" bin_path.write_text("#!/bin/sh\nexit 0\n")