Skip to content
54 changes: 39 additions & 15 deletions ingest/textbook_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@
#: cdm_eval.py::_run_subprocess_with_pgkill.
_DRAIN_TIMEOUT_S: int = 5

#: Coarsest mtime resolution a scratch directory might sit on (FAT: 2 s).
_MTIME_GRANULARITY_S: float = 2.0

#: Platform on which RLIMIT_AS is enforceable. Verified live test on
#: Darwin 25.4.0 / Apple M4 Max: setrlimit(RLIMIT_AS, (4GB, 4GB)) raises
#: ValueError because the macOS kernel keeps the hard limit at
Expand Down Expand Up @@ -331,47 +334,65 @@ def _tail(s: str, n: int = _OUTPUT_TAIL_BYTES) -> str:
return encoded[-n:].decode("utf-8", errors="replace")


def _locate_outputs(output_dir: Path, pdf_stem: str) -> tuple[Path, Path]:
def _locate_outputs(
output_dir: Path,
pdf_stem: str,
*,
written_after: float | None = None,
) -> tuple[Path, Path]:
"""Find the markdown + content_list.json that MinerU produced.

MinerU 3.2.0 with ``-b pipeline -m auto`` produces:
output_dir/<pdf_stem>/auto/<pdf_stem>.md
output_dir/<pdf_stem>/auto/<pdf_stem>_content_list.json

Per research-synthesis §D3: probe the documented path first,
fall back to a glob, raise RuntimeError if no markdown found
(FM-4 mitigation).
Probe the documented path first, fall back to a glob, and raise
RuntimeError if no markdown is found. The fallback only accepts an
md + content_list pair that shares a parent directory, never files
from different parse subdirectories.

The fallback ONLY accepts md+content_list pairs that share a
parent directory — never pairs files from different parse
subdirectories. Closes m5 adversary F4.
``written_after`` is a wall-clock time: files modified before it are
ignored. The scratch directory is reused for every parse of the same
paper, so without it a run that exits 0 but writes nothing would
hand back the previous run's output as its own.
"""

def fresh(*paths: Path) -> bool:
return written_after is None or all(
path.stat().st_mtime >= written_after for path in paths
)

direct_md = output_dir / pdf_stem / "auto" / f"{pdf_stem}.md"
direct_cl = (
output_dir / pdf_stem / "auto" / f"{pdf_stem}_content_list.json"
)
if direct_md.is_file() and direct_cl.is_file():
if direct_md.is_file() and direct_cl.is_file() and fresh(direct_md, direct_cl):
return direct_md, direct_cl

md_candidates = sorted(output_dir.rglob("*.md"))
all_md = sorted(output_dir.rglob("*.md"))
md_candidates = [md for md in all_md if fresh(md)]
if not md_candidates:
if all_md:
raise RuntimeError(
f"MinerU exited 0 but wrote no new markdown under "
f"{output_dir!s}; the {len(all_md)} .md file(s) there are "
f"left over from an earlier parse and were not used."
)
listing = sorted(p.relative_to(output_dir) for p in output_dir.rglob("*"))
raise RuntimeError(
f"MinerU exited 0 but no .md file found under "
f"{output_dir!s}. Directory contents: {listing!r}. The "
f"MinerU output-tree convention may have changed; see "
f"research-synthesis §D3 (FM-4)."
f"MinerU output-tree convention may have changed."
)
# Closes m5 F4: pair-match in same parent dir, do not cross-pair.
for md in md_candidates:
cl = md.with_name(f"{md.stem}_content_list.json")
if cl.is_file():
if cl.is_file() and fresh(cl):
return md, cl
raise RuntimeError(
f"MinerU exited 0 and produced markdown candidates "
f"({[str(p.relative_to(output_dir)) for p in md_candidates]!r}) "
f"but no paired _content_list.json in the same parent dir for "
f"any of them. Output tree partial — m6 cannot consume."
f"any of them. Output tree partial — the renderer cannot consume it."
)


Expand Down Expand Up @@ -542,6 +563,9 @@ def run_mineru_sandboxed(
)

started = time.monotonic()
# Wall clock, for comparing against output mtimes; less a margin for
# filesystems that store mtimes coarsely (FAT keeps 2 s).
written_after = time.time() - _MTIME_GRANULARITY_S
proc = subprocess.Popen(
cmd,
cwd=str(output_dir),
Expand Down Expand Up @@ -594,7 +618,7 @@ def run_mineru_sandboxed(
)

markdown_path, content_list_path = _locate_outputs(
output_dir, pdf_path.stem,
output_dir, pdf_path.stem, written_after=written_after,
)

logger.info(
Expand Down
34 changes: 31 additions & 3 deletions ops/checkpoint_notebooks_db.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""TRUNCATE-checkpoint notebooks.db WAL before a restic backup.
"""TRUNCATE-checkpoint SQLite WAL files before a restic backup.

notebook-ops-hardening-m1.
Every SQLite database in the backup set runs in WAL mode (see
:func:`backup_set_databases`). The rest of this note uses
``notebooks.db`` as the example.

``notebooks.db`` runs in WAL mode (``journal_mode=WAL`` +
``synchronous=FULL`` + ``fullfsync=ON`` from notebook-ops-hardening-m2),
Expand Down Expand Up @@ -114,10 +116,36 @@ def checkpoint(
return last


def backup_set_databases(var_root: Path) -> list[Path]:
"""Every SQLite database the backup copies, under ``var_root``.

That is ``cache/notebooks.db`` plus each notebook's stores under
``notebooks/`` (``documents.db``, ``paper_metadata.db``). Each runs in
WAL mode, so each needs the same checkpoint. A notebook's
``cache/retrieval.db`` is a regenerable query cache that the backup
excludes, so it is skipped here too.
"""
databases = [var_root / "cache" / "notebooks.db"]
notebooks = var_root / "notebooks"
if notebooks.is_dir():
databases += sorted(
db
for db in notebooks.rglob("*.db")
if not (db.name == "retrieval.db" and db.parent.name == "cache")
)
return databases


def main(argv: list[str]) -> int:
if len(argv) == 3 and argv[1] == "--backup-set":
# One "<status> <path>" line per database, for the backup wrapper.
for db in backup_set_databases(Path(argv[2])):
print(f"{checkpoint(db)} {db}")
return 0
if len(argv) != 2:
print(
"usage: checkpoint_notebooks_db.py <path-to-notebooks.db>",
"usage: checkpoint_notebooks_db.py <path-to-db>\n"
" checkpoint_notebooks_db.py --backup-set <var/arxmcp>",
file=sys.stderr,
)
return 64
Expand Down
53 changes: 27 additions & 26 deletions ops/cron/arxmcp-backup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -131,34 +131,35 @@ BACKUP_PATHS=(
"'"${REPO_ROOT}"'/var/arxmcp/cache/notebooks.db"
)

# notebook-ops-hardening-m1 (CRITICAL): TRUNCATE-checkpoint the
# notebooks.db WAL before the snapshot so the file-level copy is
# self-consistent without -wal/-shm sidecars. The helper retries on a
# busy checkpoint. Only a CLEAN status (ok/absent/no-wal) means the main
# file alone is safe to back up. A residual busy/locked status (a reader
# held an open transaction through all retries) means committed frames
# remain ONLY in the un-backed-up -wal: a main-file-only copy taken then
# is stale OR malformed-on-restore (live-verified: "database disk image
# is malformed"). So on a degraded status we (a) ALSO back up the
# -wal/-shm sidecars for this run so the captured set can recover, and
# (b) force backup_status=partial so the sentinel flags it and `forget`
# retains the prior good snapshot. stderr is NOT discarded — the helper
# diagnostics must reach the cron/journal log.
# TRUNCATE-checkpoint the WAL of every SQLite database in the backup set
# (notebooks.db plus the documents.db and paper_metadata.db of every notebook)
# before the snapshot, so each file-level copy is self-consistent without
# its -wal/-shm sidecars. The helper retries a busy checkpoint and prints
# one "<status> <path>" line per database. Only a CLEAN status
# (ok/absent/no-wal) means the main file alone is safe to back up. A
# residual busy/locked status means committed frames remain ONLY in the
# -wal, so a main-file-only copy is stale OR malformed-on-restore
# ("database disk image is malformed"). On a degraded status we (a) ALSO
# back up the -wal/-shm sidecars of that database so the captured set can
# recover, and (b) force backup_status=partial. stderr is NOT discarded:
# the diagnostics of the helper must reach the cron/journal log.
NOTEBOOKS_DB="'"${REPO_ROOT}"'/var/arxmcp/cache/notebooks.db"
CHECKPOINT_STATUS="$(python3 "'"${REPO_ROOT}"'/ops/checkpoint_notebooks_db.py" "${NOTEBOOKS_DB}" || echo error)"
CHECKPOINT_REPORT="$(python3 "'"${REPO_ROOT}"'/ops/checkpoint_notebooks_db.py" --backup-set "'"${REPO_ROOT}"'/var/arxmcp" || echo "error ${NOTEBOOKS_DB}")"
CHECKPOINT_DEGRADED=0
case "${CHECKPOINT_STATUS}" in
ok|absent|no-wal) ;;
*)
CHECKPOINT_DEGRADED=1
echo "WARN: notebooks.db WAL checkpoint status=${CHECKPOINT_STATUS}; committed frames may remain only in the un-backed-up -wal, so the captured notebooks.db alone may be stale OR corrupt-on-restore. Backing up the -wal/-shm sidecars too and marking this backup partial." >&2
for SIDECAR in "${NOTEBOOKS_DB}-wal" "${NOTEBOOKS_DB}-shm"; do
if [ -f "${SIDECAR}" ]; then
BACKUP_PATHS+=("${SIDECAR}")
fi
done
;;
esac
while read -r CHECKPOINT_STATUS DB; do
case "${CHECKPOINT_STATUS}" in
ok|absent|no-wal) ;;
*)
CHECKPOINT_DEGRADED=1
echo "WARN: ${DB} WAL checkpoint status=${CHECKPOINT_STATUS}; committed frames may remain only in the -wal, so the captured file alone may be stale OR corrupt-on-restore. Backing up its -wal/-shm sidecars too and marking this backup partial." >&2
for SIDECAR in "${DB}-wal" "${DB}-shm"; do
if [ -f "${SIDECAR}" ]; then
BACKUP_PATHS+=("${SIDECAR}")
fi
done
;;
esac
done <<< "${CHECKPOINT_REPORT}"

# Every snapshot carries arxmcp-clean or arxmcp-partial. Retention below
# counts only clean (and pre-tag, untagged) snapshots, because restic keeps
Expand Down
28 changes: 9 additions & 19 deletions server/handlers/paper.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,31 +145,21 @@ def _unsupported_envelope(
async def handle_get_paper(
paper_id: Annotated[str, Field(min_length=1, description="arXiv paper id")],
) -> dict[str, Any]:
# agent-platform-t-inert-args-cleanup (#82): the former ``version``
# arg was accepted-and-ignored (never referenced) over a corpus that
# stores no per-version rows. Removed rather than left as a lie; an
# agent that needs a specific version parses the ``vN`` suffix it
# already carries on ``paper_id``.
# F3 fix from the E06_S03 critique: validate before using
# paper_id in a SQL-style WHERE clause.
# arXMCP#209: admit the full identifier domain that ``search_papers``
# emits. This gated on the arXiv-only validator and raised "does not
# match the arXiv id format" at a ``textbook:`` id the server had just
# handed the caller. The rows are in the same chunks table under the
# same ``paper_id`` column, reached by the same escaped predicate
# below, so there was never a data reason to refuse them.
# There is no ``version`` argument: the corpus stores no per-version
# rows, and an agent that needs one already carries the ``vN`` suffix
# on ``paper_id``.
# Validate before paper_id reaches the WHERE clause. Every id kind
# search_papers can return is admitted: textbook rows live in the same
# chunks table under the same column.
try:
admit_paper_id(paper_id, SUPPORTED_SOURCE_KINDS)
except UnsupportedSourceKind as exc:
return _cap(_unsupported_envelope(paper_id, exc))
r = get_resources()

# Filter to rows matching paper_id. LanceDB doesn't expose a
# native group-by; pull the matching rows in Arrow and aggregate
# in Python. For the 50-paper corpus the worst case is ~100s of
# rows per paper — fast enough. Project at the scanner: an
# unprojected read pulls both 1024-float vector columns (~14 KB/row)
# for every row only to count them and read two version strings.
# LanceDB has no group-by, so pull the paper's rows and aggregate
# here. Only the columns read below: without a select, every row
# carries both 1024-float embedding columns and the full body text.
arrow = (
r.chunks_table.search()
.where(f"paper_id = '{_escape(paper_id)}'", prefilter=True)
Expand Down
42 changes: 19 additions & 23 deletions server/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@
from server.health import (
router as health_router,
)
from server.model_loader import TRUST_REMOTE_CODE_ENV
from server.resources import Resources

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -408,33 +409,28 @@ def _format_unknown_arxmcp_env_var(
)


#: ``ARXMCP_*`` variables the server process reads straight from the
#: environment rather than through ``Config``. Each name is imported from
#: the module that reads it, so the two cannot drift apart.
_DIRECT_SERVER_ENV_VARS: frozenset[str] = frozenset({TRUST_REMOTE_CODE_ENV})


def _scan_unknown_arxmcp_env_vars(config: Config) -> None:
"""Reject any ``ARXMCP_*`` env var not declared on :class:`Config`.

Closes F4 from the E06_S01 critique. ``pydantic-settings``'s
``extra="forbid"`` only fires for direct ``__init__`` kwargs —
NOT for env-var input. So a typo like ``ARXMCP_BIND_HOST_TYPO``
is silently ignored at the Pydantic layer. This scan walks
``os.environ`` for every ``ARXMCP_*`` key and asserts it maps to
a declared field. (The original docstring also cited
``ARXMCP_OTEL_ENDPOINT`` as an example of the "documented-but-
unimplemented" class — m1 critique F2: ``ARXMCP_OTEL_ENDPOINT``
is now declared on ``Config`` (E14 observability landed) and
would be accepted, so the example was retired.)

``onboarding-uplift-m1`` replaces the prior 30-line declared-var
dump with per-var hints: an ingest-tool carve-out for
``ARXMCP_CONTACT_EMAIL`` (the documented footgun the m1 doc sweep
removed from CLAUDE.md / README.md / Makefile), a
``difflib.get_close_matches`` (``n=1, cutoff=0.7``) suggestion for
typos like ``ARXMCP_BIND_HOTS``, and a short-form "nearest 3"
fallback otherwise. The dynamic ``Config.model_fields`` source is
preserved — adding a new ``Config`` field automatically extends
``declared`` (m1 synthesis FM-5 mitigation).
"""Reject any ``ARXMCP_*`` env var the server does not read.

``pydantic-settings``'s ``extra="forbid"`` only fires for direct
``__init__`` kwargs, not for env-var input, so a typo like
``ARXMCP_BIND_HOST_TYPO`` would otherwise be silently ignored. This
scan walks ``os.environ`` and accepts a name only if it is a
``Config`` field (derived from ``Config.model_fields``, so a new
field extends it automatically) or one of
:data:`_DIRECT_SERVER_ENV_VARS`. Rejected names get a hint: a
tailored one for known ingest-tool vars, a close-match suggestion
for typos, or the three nearest declared names.
"""
declared = frozenset(
f"ARXMCP_{name.upper()}" for name in Config.model_fields
)
) | _DIRECT_SERVER_ENV_VARS
unknown = sorted(
env_name
for env_name in os.environ
Expand Down
9 changes: 6 additions & 3 deletions server/model_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,9 @@ def validate_model_revision(revision: str, *, model_name: str) -> None:
#: refuses values other than the literal string ``"1"`` so the
#: operator must opt in explicitly (no fuzzy truthiness like
#: ``True`` / ``yes`` / ``on``).
_TRUST_REMOTE_CODE_ENV = "ARXMCP_TRUST_REMOTE_CODE"
#: The server process reads it here, not through ``Config``, so
#: ``server.main``'s unknown-variable scan imports this name to accept it.
TRUST_REMOTE_CODE_ENV = "ARXMCP_TRUST_REMOTE_CODE"


def resolve_trust_remote_code() -> bool:
Expand All @@ -107,14 +109,14 @@ def resolve_trust_remote_code() -> bool:
"``trust_remote_code=False`` is the default; enabling it requires
``ARXMCP_TRUST_REMOTE_CODE=1`` and logs a WARN."
"""
raw = os.environ.get(_TRUST_REMOTE_CODE_ENV, "")
raw = os.environ.get(TRUST_REMOTE_CODE_ENV, "")
if raw == "1":
logger.warning(
"%s=1: model loaders will pass trust_remote_code=True. "
"This re-enables the modeling-script RCE vector from "
"Threat 6 (.claude/notes/08-security-observability-ops.md). "
"Production deployments must leave this unset.",
_TRUST_REMOTE_CODE_ENV,
TRUST_REMOTE_CODE_ENV,
)
return True
return False
Expand Down Expand Up @@ -194,6 +196,7 @@ def assert_no_bin_in_snapshot(model_name: str, revision: str) -> None:
__all__ = [
"ModelPinningError",
"assert_no_bin_in_snapshot",
"TRUST_REMOTE_CODE_ENV",
"resolve_trust_remote_code",
"validate_model_revision",
]
Loading
Loading