diff --git a/ingest/textbook_parser.py b/ingest/textbook_parser.py index 0cb70067..b7a365ac 100644 --- a/ingest/textbook_parser.py +++ b/ingest/textbook_parser.py @@ -101,6 +101,9 @@ #: cdm_eval.py::_run_subprocess_with_pgkill. _DRAIN_TIMEOUT_S: int = 5 +#: Coarsest mtime resolution a scratch directory might sit on (FAT: 2 s). +_MTIME_GRANULARITY_S: float = 2.0 + #: Platform on which RLIMIT_AS is enforceable. Verified live test on #: Darwin 25.4.0 / Apple M4 Max: setrlimit(RLIMIT_AS, (4GB, 4GB)) raises #: ValueError because the macOS kernel keeps the hard limit at @@ -331,47 +334,65 @@ def _tail(s: str, n: int = _OUTPUT_TAIL_BYTES) -> str: return encoded[-n:].decode("utf-8", errors="replace") -def _locate_outputs(output_dir: Path, pdf_stem: str) -> tuple[Path, Path]: +def _locate_outputs( + output_dir: Path, + pdf_stem: str, + *, + written_after: float | None = None, +) -> tuple[Path, Path]: """Find the markdown + content_list.json that MinerU produced. MinerU 3.2.0 with ``-b pipeline -m auto`` produces: output_dir//auto/.md output_dir//auto/_content_list.json - Per research-synthesis §D3: probe the documented path first, - fall back to a glob, raise RuntimeError if no markdown found - (FM-4 mitigation). + Probe the documented path first, fall back to a glob, and raise + RuntimeError if no markdown is found. The fallback only accepts an + md + content_list pair that shares a parent directory, never files + from different parse subdirectories. - The fallback ONLY accepts md+content_list pairs that share a - parent directory — never pairs files from different parse - subdirectories. Closes m5 adversary F4. + ``written_after`` is a wall-clock time: files modified before it are + ignored. The scratch directory is reused for every parse of the same + paper, so without it a run that exits 0 but writes nothing would + hand back the previous run's output as its own. """ + + def fresh(*paths: Path) -> bool: + return written_after is None or all( + path.stat().st_mtime >= written_after for path in paths + ) + direct_md = output_dir / pdf_stem / "auto" / f"{pdf_stem}.md" direct_cl = ( output_dir / pdf_stem / "auto" / f"{pdf_stem}_content_list.json" ) - if direct_md.is_file() and direct_cl.is_file(): + if direct_md.is_file() and direct_cl.is_file() and fresh(direct_md, direct_cl): return direct_md, direct_cl - md_candidates = sorted(output_dir.rglob("*.md")) + all_md = sorted(output_dir.rglob("*.md")) + md_candidates = [md for md in all_md if fresh(md)] if not md_candidates: + if all_md: + raise RuntimeError( + f"MinerU exited 0 but wrote no new markdown under " + f"{output_dir!s}; the {len(all_md)} .md file(s) there are " + f"left over from an earlier parse and were not used." + ) listing = sorted(p.relative_to(output_dir) for p in output_dir.rglob("*")) raise RuntimeError( f"MinerU exited 0 but no .md file found under " f"{output_dir!s}. Directory contents: {listing!r}. The " - f"MinerU output-tree convention may have changed; see " - f"research-synthesis §D3 (FM-4)." + f"MinerU output-tree convention may have changed." ) - # Closes m5 F4: pair-match in same parent dir, do not cross-pair. for md in md_candidates: cl = md.with_name(f"{md.stem}_content_list.json") - if cl.is_file(): + if cl.is_file() and fresh(cl): return md, cl raise RuntimeError( f"MinerU exited 0 and produced markdown candidates " f"({[str(p.relative_to(output_dir)) for p in md_candidates]!r}) " f"but no paired _content_list.json in the same parent dir for " - f"any of them. Output tree partial — m6 cannot consume." + f"any of them. Output tree partial — the renderer cannot consume it." ) @@ -542,6 +563,9 @@ def run_mineru_sandboxed( ) started = time.monotonic() + # Wall clock, for comparing against output mtimes; less a margin for + # filesystems that store mtimes coarsely (FAT keeps 2 s). + written_after = time.time() - _MTIME_GRANULARITY_S proc = subprocess.Popen( cmd, cwd=str(output_dir), @@ -594,7 +618,7 @@ def run_mineru_sandboxed( ) markdown_path, content_list_path = _locate_outputs( - output_dir, pdf_path.stem, + output_dir, pdf_path.stem, written_after=written_after, ) logger.info( diff --git a/ops/checkpoint_notebooks_db.py b/ops/checkpoint_notebooks_db.py index c6dcee8a..be4b0e82 100644 --- a/ops/checkpoint_notebooks_db.py +++ b/ops/checkpoint_notebooks_db.py @@ -1,6 +1,8 @@ -"""TRUNCATE-checkpoint notebooks.db WAL before a restic backup. +"""TRUNCATE-checkpoint SQLite WAL files before a restic backup. -notebook-ops-hardening-m1. +Every SQLite database in the backup set runs in WAL mode (see +:func:`backup_set_databases`). The rest of this note uses +``notebooks.db`` as the example. ``notebooks.db`` runs in WAL mode (``journal_mode=WAL`` + ``synchronous=FULL`` + ``fullfsync=ON`` from notebook-ops-hardening-m2), @@ -114,10 +116,36 @@ def checkpoint( return last +def backup_set_databases(var_root: Path) -> list[Path]: + """Every SQLite database the backup copies, under ``var_root``. + + That is ``cache/notebooks.db`` plus each notebook's stores under + ``notebooks/`` (``documents.db``, ``paper_metadata.db``). Each runs in + WAL mode, so each needs the same checkpoint. A notebook's + ``cache/retrieval.db`` is a regenerable query cache that the backup + excludes, so it is skipped here too. + """ + databases = [var_root / "cache" / "notebooks.db"] + notebooks = var_root / "notebooks" + if notebooks.is_dir(): + databases += sorted( + db + for db in notebooks.rglob("*.db") + if not (db.name == "retrieval.db" and db.parent.name == "cache") + ) + return databases + + def main(argv: list[str]) -> int: + if len(argv) == 3 and argv[1] == "--backup-set": + # One " " line per database, for the backup wrapper. + for db in backup_set_databases(Path(argv[2])): + print(f"{checkpoint(db)} {db}") + return 0 if len(argv) != 2: print( - "usage: checkpoint_notebooks_db.py ", + "usage: checkpoint_notebooks_db.py \n" + " checkpoint_notebooks_db.py --backup-set ", file=sys.stderr, ) return 64 diff --git a/ops/cron/arxmcp-backup.sh b/ops/cron/arxmcp-backup.sh index 4890d5a4..f30052fe 100755 --- a/ops/cron/arxmcp-backup.sh +++ b/ops/cron/arxmcp-backup.sh @@ -131,34 +131,35 @@ BACKUP_PATHS=( "'"${REPO_ROOT}"'/var/arxmcp/cache/notebooks.db" ) -# notebook-ops-hardening-m1 (CRITICAL): TRUNCATE-checkpoint the -# notebooks.db WAL before the snapshot so the file-level copy is -# self-consistent without -wal/-shm sidecars. The helper retries on a -# busy checkpoint. Only a CLEAN status (ok/absent/no-wal) means the main -# file alone is safe to back up. A residual busy/locked status (a reader -# held an open transaction through all retries) means committed frames -# remain ONLY in the un-backed-up -wal: a main-file-only copy taken then -# is stale OR malformed-on-restore (live-verified: "database disk image -# is malformed"). So on a degraded status we (a) ALSO back up the -# -wal/-shm sidecars for this run so the captured set can recover, and -# (b) force backup_status=partial so the sentinel flags it and `forget` -# retains the prior good snapshot. stderr is NOT discarded — the helper -# diagnostics must reach the cron/journal log. +# TRUNCATE-checkpoint the WAL of every SQLite database in the backup set +# (notebooks.db plus the documents.db and paper_metadata.db of every notebook) +# before the snapshot, so each file-level copy is self-consistent without +# its -wal/-shm sidecars. The helper retries a busy checkpoint and prints +# one " " line per database. Only a CLEAN status +# (ok/absent/no-wal) means the main file alone is safe to back up. A +# residual busy/locked status means committed frames remain ONLY in the +# -wal, so a main-file-only copy is stale OR malformed-on-restore +# ("database disk image is malformed"). On a degraded status we (a) ALSO +# back up the -wal/-shm sidecars of that database so the captured set can +# recover, and (b) force backup_status=partial. stderr is NOT discarded: +# the diagnostics of the helper must reach the cron/journal log. NOTEBOOKS_DB="'"${REPO_ROOT}"'/var/arxmcp/cache/notebooks.db" -CHECKPOINT_STATUS="$(python3 "'"${REPO_ROOT}"'/ops/checkpoint_notebooks_db.py" "${NOTEBOOKS_DB}" || echo error)" +CHECKPOINT_REPORT="$(python3 "'"${REPO_ROOT}"'/ops/checkpoint_notebooks_db.py" --backup-set "'"${REPO_ROOT}"'/var/arxmcp" || echo "error ${NOTEBOOKS_DB}")" CHECKPOINT_DEGRADED=0 -case "${CHECKPOINT_STATUS}" in - ok|absent|no-wal) ;; - *) - CHECKPOINT_DEGRADED=1 - echo "WARN: notebooks.db WAL checkpoint status=${CHECKPOINT_STATUS}; committed frames may remain only in the un-backed-up -wal, so the captured notebooks.db alone may be stale OR corrupt-on-restore. Backing up the -wal/-shm sidecars too and marking this backup partial." >&2 - for SIDECAR in "${NOTEBOOKS_DB}-wal" "${NOTEBOOKS_DB}-shm"; do - if [ -f "${SIDECAR}" ]; then - BACKUP_PATHS+=("${SIDECAR}") - fi - done - ;; -esac +while read -r CHECKPOINT_STATUS DB; do + case "${CHECKPOINT_STATUS}" in + ok|absent|no-wal) ;; + *) + CHECKPOINT_DEGRADED=1 + echo "WARN: ${DB} WAL checkpoint status=${CHECKPOINT_STATUS}; committed frames may remain only in the -wal, so the captured file alone may be stale OR corrupt-on-restore. Backing up its -wal/-shm sidecars too and marking this backup partial." >&2 + for SIDECAR in "${DB}-wal" "${DB}-shm"; do + if [ -f "${SIDECAR}" ]; then + BACKUP_PATHS+=("${SIDECAR}") + fi + done + ;; + esac +done <<< "${CHECKPOINT_REPORT}" # Every snapshot carries arxmcp-clean or arxmcp-partial. Retention below # counts only clean (and pre-tag, untagged) snapshots, because restic keeps diff --git a/server/handlers/paper.py b/server/handlers/paper.py index 4d930a74..70f6ecc3 100644 --- a/server/handlers/paper.py +++ b/server/handlers/paper.py @@ -145,31 +145,21 @@ def _unsupported_envelope( async def handle_get_paper( paper_id: Annotated[str, Field(min_length=1, description="arXiv paper id")], ) -> dict[str, Any]: - # agent-platform-t-inert-args-cleanup (#82): the former ``version`` - # arg was accepted-and-ignored (never referenced) over a corpus that - # stores no per-version rows. Removed rather than left as a lie; an - # agent that needs a specific version parses the ``vN`` suffix it - # already carries on ``paper_id``. - # F3 fix from the E06_S03 critique: validate before using - # paper_id in a SQL-style WHERE clause. - # arXMCP#209: admit the full identifier domain that ``search_papers`` - # emits. This gated on the arXiv-only validator and raised "does not - # match the arXiv id format" at a ``textbook:`` id the server had just - # handed the caller. The rows are in the same chunks table under the - # same ``paper_id`` column, reached by the same escaped predicate - # below, so there was never a data reason to refuse them. + # There is no ``version`` argument: the corpus stores no per-version + # rows, and an agent that needs one already carries the ``vN`` suffix + # on ``paper_id``. + # Validate before paper_id reaches the WHERE clause. Every id kind + # search_papers can return is admitted: textbook rows live in the same + # chunks table under the same column. try: admit_paper_id(paper_id, SUPPORTED_SOURCE_KINDS) except UnsupportedSourceKind as exc: return _cap(_unsupported_envelope(paper_id, exc)) r = get_resources() - # Filter to rows matching paper_id. LanceDB doesn't expose a - # native group-by; pull the matching rows in Arrow and aggregate - # in Python. For the 50-paper corpus the worst case is ~100s of - # rows per paper — fast enough. Project at the scanner: an - # unprojected read pulls both 1024-float vector columns (~14 KB/row) - # for every row only to count them and read two version strings. + # LanceDB has no group-by, so pull the paper's rows and aggregate + # here. Only the columns read below: without a select, every row + # carries both 1024-float embedding columns and the full body text. arrow = ( r.chunks_table.search() .where(f"paper_id = '{_escape(paper_id)}'", prefilter=True) diff --git a/server/main.py b/server/main.py index b77be7b6..8a603295 100644 --- a/server/main.py +++ b/server/main.py @@ -91,6 +91,7 @@ from server.health import ( router as health_router, ) +from server.model_loader import TRUST_REMOTE_CODE_ENV from server.resources import Resources logger = logging.getLogger(__name__) @@ -408,33 +409,28 @@ def _format_unknown_arxmcp_env_var( ) +#: ``ARXMCP_*`` variables the server process reads straight from the +#: environment rather than through ``Config``. Each name is imported from +#: the module that reads it, so the two cannot drift apart. +_DIRECT_SERVER_ENV_VARS: frozenset[str] = frozenset({TRUST_REMOTE_CODE_ENV}) + + def _scan_unknown_arxmcp_env_vars(config: Config) -> None: - """Reject any ``ARXMCP_*`` env var not declared on :class:`Config`. - - Closes F4 from the E06_S01 critique. ``pydantic-settings``'s - ``extra="forbid"`` only fires for direct ``__init__`` kwargs — - NOT for env-var input. So a typo like ``ARXMCP_BIND_HOST_TYPO`` - is silently ignored at the Pydantic layer. This scan walks - ``os.environ`` for every ``ARXMCP_*`` key and asserts it maps to - a declared field. (The original docstring also cited - ``ARXMCP_OTEL_ENDPOINT`` as an example of the "documented-but- - unimplemented" class — m1 critique F2: ``ARXMCP_OTEL_ENDPOINT`` - is now declared on ``Config`` (E14 observability landed) and - would be accepted, so the example was retired.) - - ``onboarding-uplift-m1`` replaces the prior 30-line declared-var - dump with per-var hints: an ingest-tool carve-out for - ``ARXMCP_CONTACT_EMAIL`` (the documented footgun the m1 doc sweep - removed from CLAUDE.md / README.md / Makefile), a - ``difflib.get_close_matches`` (``n=1, cutoff=0.7``) suggestion for - typos like ``ARXMCP_BIND_HOTS``, and a short-form "nearest 3" - fallback otherwise. The dynamic ``Config.model_fields`` source is - preserved — adding a new ``Config`` field automatically extends - ``declared`` (m1 synthesis FM-5 mitigation). + """Reject any ``ARXMCP_*`` env var the server does not read. + + ``pydantic-settings``'s ``extra="forbid"`` only fires for direct + ``__init__`` kwargs, not for env-var input, so a typo like + ``ARXMCP_BIND_HOST_TYPO`` would otherwise be silently ignored. This + scan walks ``os.environ`` and accepts a name only if it is a + ``Config`` field (derived from ``Config.model_fields``, so a new + field extends it automatically) or one of + :data:`_DIRECT_SERVER_ENV_VARS`. Rejected names get a hint: a + tailored one for known ingest-tool vars, a close-match suggestion + for typos, or the three nearest declared names. """ declared = frozenset( f"ARXMCP_{name.upper()}" for name in Config.model_fields - ) + ) | _DIRECT_SERVER_ENV_VARS unknown = sorted( env_name for env_name in os.environ diff --git a/server/model_loader.py b/server/model_loader.py index 64abf4fc..07c84fb3 100644 --- a/server/model_loader.py +++ b/server/model_loader.py @@ -93,7 +93,9 @@ def validate_model_revision(revision: str, *, model_name: str) -> None: #: refuses values other than the literal string ``"1"`` so the #: operator must opt in explicitly (no fuzzy truthiness like #: ``True`` / ``yes`` / ``on``). -_TRUST_REMOTE_CODE_ENV = "ARXMCP_TRUST_REMOTE_CODE" +#: The server process reads it here, not through ``Config``, so +#: ``server.main``'s unknown-variable scan imports this name to accept it. +TRUST_REMOTE_CODE_ENV = "ARXMCP_TRUST_REMOTE_CODE" def resolve_trust_remote_code() -> bool: @@ -107,14 +109,14 @@ def resolve_trust_remote_code() -> bool: "``trust_remote_code=False`` is the default; enabling it requires ``ARXMCP_TRUST_REMOTE_CODE=1`` and logs a WARN." """ - raw = os.environ.get(_TRUST_REMOTE_CODE_ENV, "") + raw = os.environ.get(TRUST_REMOTE_CODE_ENV, "") if raw == "1": logger.warning( "%s=1: model loaders will pass trust_remote_code=True. " "This re-enables the modeling-script RCE vector from " "Threat 6 (.claude/notes/08-security-observability-ops.md). " "Production deployments must leave this unset.", - _TRUST_REMOTE_CODE_ENV, + TRUST_REMOTE_CODE_ENV, ) return True return False @@ -194,6 +196,7 @@ def assert_no_bin_in_snapshot(model_name: str, revision: str) -> None: __all__ = [ "ModelPinningError", "assert_no_bin_in_snapshot", + "TRUST_REMOTE_CODE_ENV", "resolve_trust_remote_code", "validate_model_revision", ] diff --git a/server/parse_tracker.py b/server/parse_tracker.py index b800580d..f5b8b378 100644 --- a/server/parse_tracker.py +++ b/server/parse_tracker.py @@ -26,7 +26,9 @@ see ``pending`` indefinitely. - **``done_callback`` removes the tracker entry** but the DB row update is INSIDE the worker function so terminal-state writes - happen even if the callback is delayed. + happen even if the callback is delayed. Every path out of the + worker attempts a terminal write: a row left ``running`` blocks + every later upload to that notebook until a restart. - **Lifespan shutdown cancels in-flight tasks** then a startup-time ``mark_orphaned_parses_failed()`` sweep cleans up any leftover ``running`` rows from a hard crash (synthesis FM-4). @@ -96,6 +98,11 @@ def _redact_path_prefix(text: str) -> str: #: an absolute-path prefix (leading ``/`` or ``\``) up to and #: including ``var/arxmcp/`` and replaces it with the bare relative #: form. Non-greedy so only the prefix is consumed. +#: Seconds to wait before each retry of the success-path status write. +#: A failure there is most likely SQLite ``database is locked`` under a +#: concurrent status poll, which clears within moments. +_COMPLETE_WRITE_RETRY_DELAYS_S: tuple[float, ...] = (0.25, 1.0) + _ABS_PATH_PREFIX_RE = re.compile(r"[/\\][\w /\\.\-]*?var[/\\]arxmcp[/\\]") @@ -206,6 +213,29 @@ def _on_task_done(self, slug: str, task: asyncio.Task) -> None: "parse task for slug=%s raised: %s", slug, exc, ) + @staticmethod + async def _record_complete( + store: NotebooksStore, slug: str, html_path_str: str, + ) -> None: + """Write ``complete``, retrying a transient failure.""" + for delay in (*_COMPLETE_WRITE_RETRY_DELAYS_S, None): + try: + await store.update_parse_status( + slug, + store.PARSE_STATUS_COMPLETE, + parse_error="", + parsed_html_path=html_path_str, + ) + return + except Exception: + if delay is None: + raise + logger.warning( + "recording parse complete for slug=%s failed; " + "retrying in %.2fs", slug, delay, exc_info=True, + ) + await asyncio.sleep(delay) + async def _run_parse( self, *, @@ -279,17 +309,30 @@ async def _run_parse( return # Success: record the parsed_html_path scrubbed to its - # ``var/arxmcp/``-relative form (m6 F1). Storing the - # absolute path would leak the operator's home dir - # through the /parse-status JSON — the m9 redact_paths - # discipline applies here too. - html_path_str = redact_html_path(render_result.output_html_path) - await store.update_parse_status( - slug, - store.PARSE_STATUS_COMPLETE, - parse_error="", - parsed_html_path=html_path_str, - ) + # ``var/arxmcp/``-relative form. Storing the absolute path + # would leak the operator's home dir through the + # /parse-status JSON. + try: + html_path_str = redact_html_path(render_result.output_html_path) + await self._record_complete(store, slug, html_path_str) + except Exception as e: # noqa: BLE001 + # The HTML is on disk, but a row left ``running`` would + # block every later upload to this notebook, so settle + # for a terminal ``failed`` that says what happened. + logger.exception( + "parse for slug=%s finished but its status could not " + "be recorded", slug, + ) + with contextlib.suppress(Exception): + await store.update_parse_status( + slug, + store.PARSE_STATUS_FAILED, + parse_error=_format_parse_error( + "parse finished but recording it failed: " + f"{type(e).__name__}: {e}" + ), + ) + return logger.info( "parse complete: slug=%s html=%s errors=%d " "mineru_wall=%.1fs render_wall=%.2fs", diff --git a/server/tools.py b/server/tools.py index c76d0de4..e47975d7 100644 --- a/server/tools.py +++ b/server/tools.py @@ -872,7 +872,7 @@ def cap_result_list( list_key: str, chunk_id: str | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: - """E13_S04b — cap the byte size of a multi-result tool envelope. + """Cap the byte size of a multi-result tool envelope. The companion to :func:`enforce_byte_cap` for handlers whose over-cap surface is the aggregate response (a list of rows in @@ -882,14 +882,10 @@ def cap_result_list( :func:`enforce_byte_cap` signature so callers can treat both helpers uniformly. - Closes the F1 finding from the E13_S04b adversary critique: - multi-result tools previously called ``enforce_byte_cap(payload)`` - with the default ``body_text_path=("body_text",)``, which silently - no-ops on payloads with no top-level ``body_text`` key. - ``body_truncated=True`` was set but the payload bytes were - unchanged — a 500 KB response stayed 500 KB on the wire. + :func:`enforce_byte_cap` cannot do this job: it trims a top-level + ``body_text`` and silently no-ops on payloads without one. - This helper iteratively pops the trailing row from + This helper drops trailing rows from ``structured_content[list_key]`` until the serialized payload (with the :data:`_WIRE_OVERHEAD_FACTOR` wire-overhead correction) fits under @@ -920,20 +916,29 @@ def cap_result_list( """ cap = get_resources().config.result_byte_cap - def _measure(d: dict[str, Any]) -> int: + def _nbytes(value: Any) -> int: return len( - json.dumps(d, ensure_ascii=False, sort_keys=True).encode("utf-8") - ) * _WIRE_OVERHEAD_FACTOR + json.dumps(value, ensure_ascii=False, sort_keys=True).encode("utf-8") + ) - if _measure(structured_content) <= cap: + size = _nbytes(structured_content) + if size * _WIRE_OVERHEAD_FACTOR <= cap: return structured_content, [] # Over cap — deep-copy so we never mutate the caller's payload. truncated = json.loads(json.dumps(structured_content)) rows = truncated.get(list_key) if isinstance(rows, list): - while rows and _measure(truncated) > cap: + # Serialize each row once and subtract as rows go, rather than + # re-serializing the whole payload per dropped row. With the + # default separators a nested value serializes to the same bytes + # it occupies inside its parent, so the arithmetic is exact: a + # row costs its own bytes plus the ", " before it, and the last + # remaining row has no separator. + row_sizes = [_nbytes(row) for row in rows] + while rows and size * _WIRE_OVERHEAD_FACTOR > cap: rows.pop() + size -= row_sizes.pop() + (2 if rows else 0) truncated["body_truncated"] = True blocks: list[dict[str, Any]] = [] if chunk_id is not None: diff --git a/tests/eval/test_equations_parity.py b/tests/eval/test_equations_parity.py index 3c7e7b97..0ee91a9f 100644 --- a/tests/eval/test_equations_parity.py +++ b/tests/eval/test_equations_parity.py @@ -1,6 +1,6 @@ -"""retrieval-unlocks-m4 — the equations-fixture parity gate. +"""The equations-fixture parity gate. -m4's acceptance is conditional: the TED route is reported for a LaTeX +The LaTeX route's acceptance is conditional: the TED route is reported for a LaTeX query *"when the equations-fixture parity check has passed"*. This is that check, kept as a regression guard so a future ``latex2mathml`` bump (or a change to ``parse_mathml_to_tree``) cannot silently degrade the @@ -60,7 +60,9 @@ pytestmark = pytest.mark.eval -_CORPUS = Path("var/arxmcp/corpus/parsed") +# Anchored to the repo root, not the working directory, so the gate runs +# (or skips) the same way whichever directory pytest is started from. +_CORPUS = Path(__file__).resolve().parents[2] / "var" / "arxmcp" / "corpus" / "parsed" #: Gate. Below this, the LaTeX route is not safe to leave enabled and #: ``Config.eq_latex_route`` should default False (or the pooled @@ -108,8 +110,7 @@ def corpus_pairs(): if len(pairs) < _POOL: pytest.skip( f"parsed corpus absent or too small ({len(pairs)} equations; " - f"need {_POOL}). Populate var/arxmcp/corpus/parsed/ to run " - f"the m4 parity gate." + f"need {_POOL}). Populate {_CORPUS} to run the parity gate." ) random.Random(7).shuffle(pairs) return pairs @@ -210,8 +211,8 @@ def test_canonicalization_erases_parallel_markup(self): Without this, every stored tree carries a Content-MathML annotation subtree the query side never has, and parity is - impossible regardless of converter quality — this was the m4 - blocker, and it also silently degraded the already-shipped + impossible regardless of converter quality — this was the LaTeX + route's blocker, and it also silently degraded the already-shipped MathML-input route. """ latexml = ( @@ -247,13 +248,13 @@ def test_font_variants_collapse_KNOWN_LIMITATION(self): are indistinguishable. This is a DELIBERATE, documented loss, not a regression: the stored corpus already collapses them, because LaTeXML encodes the variant in a ``mathvariant`` attribute that - parse_mathml_to_tree has dropped since E10_S03. NFKC only makes + parse_mathml_to_tree has always dropped. NFKC only makes the query side match that pre-existing collapse. This test PINS the known behaviour so a future change that (accidentally or deliberately) starts distinguishing variants is a conscious decision with a failing test, not a silent surprise. See ``_fold_text`` for the full rationale and the path to fixing it - (preserve mathvariant on both sides — an E10_S03-scope re-index).""" + (preserve mathvariant on both sides — a full re-index).""" blackboard = "ℂ" # ℂ (U+2102) script = "𝒞" # 𝒞 (U+1D49E) plain = "C" diff --git a/tests/test_backup_wrapper.py b/tests/test_backup_wrapper.py index 70b7f998..cb10a3fb 100644 --- a/tests/test_backup_wrapper.py +++ b/tests/test_backup_wrapper.py @@ -227,8 +227,7 @@ def test_degraded_checkpoint_marks_partial(self): assert "CHECKPOINT_DEGRADED=1" in text # The degraded path forces partial and appends the WAL sidecars. assert 'BACKUP_PATHS+=("${SIDECAR}")' in text - assert "${NOTEBOOKS_DB}-wal" in text - assert "${NOTEBOOKS_DB}-shm" in text + assert '"${DB}-wal" "${DB}-shm"' in text # And the final status is forced partial when degraded. The token # comes from the shared vocabulary (arXMCP#202), not a literal — # see ops/cron/backup-status-lib.sh and tests/ diff --git a/tests/test_cap_result_list.py b/tests/test_cap_result_list.py new file mode 100644 index 00000000..f22f6399 --- /dev/null +++ b/tests/test_cap_result_list.py @@ -0,0 +1,105 @@ +"""``cap_result_list`` drops the lowest-ranked rows until the payload fits.""" + +from __future__ import annotations + +import json +import random +from types import SimpleNamespace + +import pytest + +from server import tools +from server.tools import cap_result_list, reset_resources_for_tests, set_resources + + +def _wire_size(payload: dict) -> int: + text = json.dumps(payload, ensure_ascii=False, sort_keys=True) + return len(text.encode("utf-8")) * tools._WIRE_OVERHEAD_FACTOR + + +def _rows_that_fit(payload: dict, cap: int) -> int: + """Reference answer: the most leading rows whose payload fits.""" + rows = payload["results"] + for n in range(len(rows), -1, -1): + if _wire_size({**payload, "results": rows[:n]}) <= cap: + return n + return 0 + + +@pytest.fixture +def byte_cap(): + def install(cap: int) -> None: + set_resources(SimpleNamespace(config=SimpleNamespace(result_byte_cap=cap))) + + yield install + reset_resources_for_tests() + + +def _payload(seed: int, n_rows: int) -> dict: + rng = random.Random(seed) + alphabet = "abcxyz ∂∑ℤ→é\"\\" + rows = [ + { + "chunk_id": f"2401.{i:05d}#c{i}", + "score": rng.random(), + "snippet": "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 300))), + "meta": {"kind": rng.choice(["theorem", "proof"]), "n": [i, None, True]}, + } + for i in range(n_rows) + ] + return {"query": "zeta ∑", "results": rows, "retrieval_mode": "hybrid"} + + +@pytest.mark.parametrize("seed", range(5)) +def test_keeps_exactly_the_rows_that_fit(byte_cap, seed): + payload = _payload(seed, 40) + full = _wire_size(payload) + for cap in sorted({full // 7, full // 3, full // 2, full - 2, 50, 1}): + byte_cap(cap) + out, blocks = cap_result_list(payload, "results") + + assert len(out["results"]) == _rows_that_fit(payload, cap) + assert out["results"] == payload["results"][: len(out["results"])] + assert out["body_truncated"] is True + assert blocks == [] + + +def test_under_cap_payload_is_returned_untouched(byte_cap): + payload = _payload(0, 5) + byte_cap(_wire_size(payload)) + + out, _ = cap_result_list(payload, "results") + + assert out is payload + + +def test_caller_payload_is_not_mutated(byte_cap): + payload = _payload(1, 20) + before = json.dumps(payload, sort_keys=True) + byte_cap(_wire_size(payload) // 4) + + cap_result_list(payload, "results") + + assert json.dumps(payload, sort_keys=True) == before + + +def test_serialization_work_is_linear_in_payload_size(byte_cap, monkeypatch): + """Trimming must not re-serialize the whole payload per dropped row, + which costs bytes quadratic in the row count.""" + payload = _payload(2, 200) + payload_chars = len(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + byte_cap(200) + serialized_chars = 0 + real_dumps = json.dumps + + def counting_dumps(*args, **kwargs): + nonlocal serialized_chars + text = real_dumps(*args, **kwargs) + serialized_chars += len(text) + return text + + monkeypatch.setattr(tools.json, "dumps", counting_dumps) + out, _ = cap_result_list(payload, "results") + + assert out["results"] == [] + assert serialized_chars <= 5 * payload_chars diff --git a/tests/test_checkpoint_notebooks_db.py b/tests/test_checkpoint_notebooks_db.py index 97b7f726..2f273f25 100644 --- a/tests/test_checkpoint_notebooks_db.py +++ b/tests/test_checkpoint_notebooks_db.py @@ -150,3 +150,50 @@ def test_directory_in_place_of_db_returns_locked(self, tmp_path): status = checkpoint(fake, attempts=2, delay_s=0.0) assert status == "locked" assert status not in CLEAN_STATUSES + + +class TestBackupSet: + """``--backup-set`` checkpoints every SQLite database the backup copies, + not only notebooks.db.""" + + def test_checkpoints_every_backed_up_database(self, tmp_path, capsys): + from ops.checkpoint_notebooks_db import main + + var = tmp_path / "var" / "arxmcp" + backed_up = [ + var / "cache" / "notebooks.db", + var / "notebooks" / "demo" / "documents.db", + var / "notebooks" / "demo" / "paper_metadata.db", + ] + query_cache = var / "notebooks" / "demo" / "cache" / "retrieval.db" + writers = [] + for db in [*backed_up, query_cache]: + db.parent.mkdir(parents=True, exist_ok=True) + _wal_db_with_frames(db) + # An open, idle connection keeps committed frames in the WAL + # without blocking a checkpoint. + conn = sqlite3.connect(str(db)) + conn.execute("INSERT INTO t VALUES (2)") + conn.commit() + writers.append(conn) + try: + assert main(["checkpoint", "--backup-set", str(var)]) == 0 + report = dict( + reversed(line.split(" ", 1)) + for line in capsys.readouterr().out.splitlines() + ) + + assert report == {str(db): "ok" for db in backed_up} + for db in backed_up: + assert Path(f"{db}-wal").stat().st_size == 0 + assert Path(f"{query_cache}-wal").stat().st_size > 0 + finally: + for conn in writers: + conn.close() + + def test_missing_notebooks_db_reports_absent(self, tmp_path, capsys): + from ops.checkpoint_notebooks_db import main + + var = tmp_path / "var" / "arxmcp" + assert main(["checkpoint", "--backup-set", str(var)]) == 0 + assert capsys.readouterr().out == f"absent {var / 'cache' / 'notebooks.db'}\n" diff --git a/tests/test_parse_tracker.py b/tests/test_parse_tracker.py index 15d3b0ef..1c2dcf23 100644 --- a/tests/test_parse_tracker.py +++ b/tests/test_parse_tracker.py @@ -14,6 +14,7 @@ import asyncio import dataclasses import os +import sqlite3 import subprocess from pathlib import Path from unittest.mock import AsyncMock, MagicMock @@ -288,6 +289,75 @@ def test_dataclass_render_result_is_frozen(self) -> None: # --------------------------------------------------------------------------- +class TestSuccessPathStatusWrite: + """A finished parse must never leave its row ``running``.""" + + @staticmethod + def _run_successful_parse( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, store: MagicMock, + ) -> None: + from ingest import textbook_parser, textbook_renderer + from server import parse_tracker + + mineru_result = _make_mineru_result(tmp_path) + render_result = _make_render_result(tmp_path) + monkeypatch.setattr( + textbook_parser, "run_mineru_sandboxed", + lambda pdf_path, output_dir: mineru_result, + ) + monkeypatch.setattr( + textbook_renderer, "render_mineru_to_html", + lambda r, p, pid: render_result, + ) + monkeypatch.setattr( + parse_tracker, "_COMPLETE_WRITE_RETRY_DELAYS_S", (0, 0), raising=False, + ) + + async def _run() -> None: + tracker = ParseTaskTracker() + await tracker.start_parse( + slug="test-slug", + pdf_path=tmp_path / "x.pdf", + paper_id="textbook:x", + output_dir=tmp_path / "out", + parsed_dir=tmp_path / "parsed", + store=store, + ) + + asyncio.run(_run()) + + def test_transient_write_failure_still_records_complete( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + store = _make_store_mock() + store.update_parse_status = AsyncMock( + side_effect=[sqlite3.OperationalError("database is locked"), True], + ) + + self._run_successful_parse(tmp_path, monkeypatch, store) + + statuses = [c.args[1] for c in store.update_parse_status.call_args_list] + assert statuses == ["complete", "complete"] + + def test_persistent_write_failure_ends_failed_not_running( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + store = _make_store_mock() + + async def update(slug, status, **kwargs): + if status == "complete": + raise sqlite3.OperationalError("database is locked") + return True + + store.update_parse_status = AsyncMock(side_effect=update) + + self._run_successful_parse(tmp_path, monkeypatch, store) + + last = store.update_parse_status.call_args + assert last.args[1] == "failed" + assert "recording it failed" in last.kwargs["parse_error"] + + def _pid_alive(pid: int) -> bool: """Direct signal-0 probe. Deliberately not the code under test.""" try: diff --git a/tests/test_server_startup.py b/tests/test_server_startup.py index 061d7dd5..1bddc7cc 100644 --- a/tests/test_server_startup.py +++ b/tests/test_server_startup.py @@ -1318,6 +1318,16 @@ def test_create_app_rejects_unknown_env(self, monkeypatch, seeded_lancedb): with pytest.raises(ValueError, match="unknown ARXMCP_"): create_app(cfg) + def test_create_app_accepts_the_trust_remote_code_escape_hatch( + self, monkeypatch, seeded_lancedb + ): + """ARXMCP_TRUST_REMOTE_CODE is read by server.model_loader, not + Config. Setting it must not stop the server from being built.""" + monkeypatch.setenv("ARXMCP_TRUST_REMOTE_CODE", "1") + cfg = Config(lancedb_path=seeded_lancedb) + + create_app(cfg) + # --- onboarding-uplift-m1 (AC2 + AC9): close-match + carve-out ----- def test_close_match_suggestion_for_typo(self, monkeypatch): diff --git a/tests/test_textbook_parser.py b/tests/test_textbook_parser.py index be1f1924..a0f59596 100644 --- a/tests/test_textbook_parser.py +++ b/tests/test_textbook_parser.py @@ -24,6 +24,7 @@ import signal import subprocess import sys +import time from pathlib import Path from unittest.mock import MagicMock, patch @@ -813,6 +814,52 @@ def test_default_timeout_used_when_none( assert comm_kwargs["timeout"] == sentinel_timeout +class TestReparseIgnoresEarlierOutput: + """The scratch directory is reused for every parse of a paper, so a + run must only accept output it wrote itself.""" + + @staticmethod + def _earlier_parse(tmp_path: Path, stem: str = "book") -> tuple[Path, Path]: + pdf = tmp_path / f"{stem}.pdf" + pdf.write_bytes(b"%PDF-1.4\n%fake\n") + out_dir = tmp_path / "out" + auto = out_dir / stem / "auto" + auto.mkdir(parents=True) + an_hour_ago = time.time() - 3600 + for name, text in ((f"{stem}.md", "# Old book\n"), (f"{stem}_content_list.json", "[]")): + (auto / name).write_text(text) + os.utime(auto / name, (an_hour_ago, an_hour_ago)) + return pdf, out_dir + + def test_run_that_writes_nothing_fails( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + pdf, out_dir = self._earlier_parse(tmp_path) + monkeypatch.setenv("ARXMCP_MINERU_BIN", _create_fake_bin(tmp_path)) + with patch.object(subprocess, "Popen", return_value=_fake_proc()), \ + pytest.raises(RuntimeError, match="earlier parse"): + run_mineru_sandboxed(pdf, out_dir, timeout_s=60) + + def test_run_that_rewrites_the_output_returns_it( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + pdf, out_dir = self._earlier_parse(tmp_path) + monkeypatch.setenv("ARXMCP_MINERU_BIN", _create_fake_bin(tmp_path)) + proc = _fake_proc() + + def mineru_writes(*args, **kwargs): + auto = out_dir / "book" / "auto" + (auto / "book.md").write_text("# New book\n") + (auto / "book_content_list.json").write_text("[]") + return ("", "") + + proc.communicate.side_effect = mineru_writes + with patch.object(subprocess, "Popen", return_value=proc): + result = run_mineru_sandboxed(pdf, out_dir, timeout_s=60) + + assert result.markdown_path.read_text() == "# New book\n" + + def _create_fake_bin(tmp_path: Path) -> str: bin_path = tmp_path / "fake_mineru" bin_path.write_text("#!/bin/sh\nexit 0\n")