Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 29 additions & 26 deletions ingest/bm25_indexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
``body_tokens`` from the pinned LanceDB dataset version (E04_S02
:func:`server.corpus.open_chunks_table`) and feeding the
whitespace-split token streams to :class:`rank_bm25.BM25Okapi`. The
fitted index is pickled to ``var/arxmcp/index/bm25/v<N>/bm25.pkl`` and
fitted index is pickled to ``<lancedb>/derived/bm25/v<N>/bm25.pkl`` and
the row-aligned chunk_id list is written as ``chunk_ids.json`` next to
it. The MCP server (E07, Sonnet B) loads the pair on first lexical
query; it never enters the in-memory ingestion path.
Expand Down Expand Up @@ -54,7 +54,7 @@
**Pickle security.** ``bm25.pkl`` is produced locally from trusted
LanceDB data by this module. ``pickle.load`` on an untrusted file is
remote-code-execution; the path
``var/arxmcp/index/bm25/`` MUST be treated as trusted-local. Threat 6
the BM25 root MUST be treated as trusted-local. Threat 6
(``08-security-observability-ops.md``) covers model-weight pickles
(deny); the BM25 pickle is application data with the analogous-
narrower attack surface (trust local, deny remote).
Expand All @@ -63,7 +63,7 @@
verify file ownership matches process UID and refuse world-writable
paths before calling ``pickle.load``. The defense-in-depth concern
(M1 from the E04_S04 critique) is that documentation alone is not
enforcement. Rationale: when ``var/arxmcp/index/bm25/`` lives on a
enforcement. Rationale: when the BM25 root lives on a
shared filesystem (NFS, Docker bind mount, multi-tenant container)
an attacker who can write to that path achieves RCE in the server
process. The mitigation is a simple stat-based check at load time;
Expand Down Expand Up @@ -95,39 +95,40 @@

REPO_ROOT = Path(__file__).resolve().parent.parent

# The directory layout convention: ``var/arxmcp/index/bm25/v<N>/``.
# The directory layout convention: ``<lancedb>/derived/bm25/v<N>/``.
# Confine the ``f"v{N}"`` literal to ``_bm25_version_dir`` below — every
# other reference goes through that helper.
BM25_DIR_NAME = "bm25"
BM25_INDEX_NAME = "bm25.pkl"
BM25_CHUNK_IDS_NAME = "chunk_ids.json"
BM25_INDEX_ROOT = REPO_ROOT / "var" / "arxmcp" / "index" / BM25_DIR_NAME
BM25_STATS_PATH = REPO_ROOT / "var" / "arxmcp" / "ops" / "bm25-stats.jsonl"


def _bm25_version_dir(
corpus_version: int,
index_root: Path | None = None,
def resolve_bm25_root(
lancedb_path: str | Path, index_root: str | Path | None = None,
) -> Path:
"""Return the per-version BM25 index directory.
"""The BM25 root for the dataset at ``lancedb_path``.

``index_root`` when given, else ``derived/bm25`` inside the dataset.
A ``corpus_version`` is unique only within its dataset, so keeping the
``v<N>`` directories there means two datasets at the same version
never share an index, and cutover or rollback moves the index with
the chunks it was built from.
"""
if index_root is not None:
return Path(index_root)
from server.application_paths import derived_index_path

return derived_index_path(lancedb_path, BM25_DIR_NAME)


def _bm25_version_dir(corpus_version: int, index_root: Path) -> Path:
"""Return the per-version BM25 index directory under ``index_root``.

Confines the ``f"v{N}"`` literal to a single function so a future
rename (e.g. zero-padding to ``v00007``) is a one-line change.

Parameters
----------
corpus_version:
MVCC version integer, e.g. 1.
index_root:
Optional override for the BM25 artifact root. When ``None``
(the default), resolves :data:`BM25_INDEX_ROOT` **at call time**
so ``monkeypatch.setattr(bm25_mod, "BM25_INDEX_ROOT", ...)``
continues to intercept the path in tests (notebook-bm25-isolation-m1).
When provided (fork-C notebook mode), uses that path as the root
so the per-notebook index is isolated from the shared corpus.
"""
root = index_root if index_root is not None else BM25_INDEX_ROOT
return root / f"v{corpus_version}"
return Path(index_root) / f"v{corpus_version}"


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -270,7 +271,7 @@ def build_bm25_index(
fits a :class:`rank_bm25.BM25Okapi` over the whitespace-split token
streams, and writes ``bm25.pkl`` + ``chunk_ids.json`` to
``<index_root>/v<corpus_version>/`` atomically (where ``index_root``
defaults to :data:`BM25_INDEX_ROOT` when ``None``).
defaults to ``derived/bm25`` inside the dataset when ``None``).

.. warning::

Expand Down Expand Up @@ -324,7 +325,9 @@ def build_bm25_index(
)

start = time.monotonic()
version_dir = _bm25_version_dir(corpus_version, index_root=index_root)
version_dir = _bm25_version_dir(
corpus_version, resolve_bm25_root(lancedb_path, index_root)
)
pkl_path = version_dir / BM25_INDEX_NAME
ids_path = version_dir / BM25_CHUNK_IDS_NAME

Expand Down Expand Up @@ -452,8 +455,8 @@ def build_bm25_index(
"BM25_CHUNK_IDS_NAME",
"BM25_DIR_NAME",
"BM25_INDEX_NAME",
"BM25_INDEX_ROOT",
"BM25_STATS_PATH",
"BM25Stats",
"build_bm25_index",
"resolve_bm25_root",
]
11 changes: 5 additions & 6 deletions server/application_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
_LAYOUT = (
("corpus", "corpus"), ("index", "index"),
("lancedb", "index/lancedb"), ("kuzu", "index/kuzu"),
("bm25", "index/bm25"), ("sqlite", "index/sqlite"),
("bm25", "index/lancedb/derived/bm25"), ("sqlite", "index/sqlite"),
("theorem_names_db", "index/lancedb/derived/theorem_names.db"),
("notebooks", "notebooks"), ("cache", "cache"),
("retrieval_cache_db", "cache/retrieval.db"),
Expand Down Expand Up @@ -187,11 +187,10 @@ def resolve(
values[field] = resolved
else:
values[field] = _inside(canonical_root, path, field)
if not supplied.get(alias_by_field["theorem_names_db"]):
# Follows the dataset, including a dataset moved by its own alias.
values["theorem_names_db"] = derived_index_path(
values["lancedb"], "theorem_names.db"
)
# Derived indexes follow the dataset, including one moved by its alias.
for field, name in (("bm25", "bm25"), ("theorem_names_db", "theorem_names.db")):
if not supplied.get(alias_by_field[field]):
values[field] = derived_index_path(values["lancedb"], name)
return cls(
selected,
canonical_root,
Expand Down
51 changes: 19 additions & 32 deletions server/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,18 +149,11 @@ class Config(BaseSettings):
#: directory is created at ``Resources.startup()`` time.
cache_db_path: Path = legacy_source_path("retrieval_cache_db")

#: Per-notebook BM25 index root (notebook-bm25-isolation-m1).
#: Default ``None`` means "resolve :data:`ingest.bm25_indexer.BM25_INDEX_ROOT`
#: at call time" — preserving the conftest autouse
#: ``monkeypatch.setattr(bm25_mod, "BM25_INDEX_ROOT", ...)`` used by
#: ~40 tests. When :attr:`notebook` is set, the
#: :meth:`derive_notebook_lancedb_path` validator rewrites this to
#: ``var/arxmcp/notebooks/<slug>/index/bm25`` (mirroring the
#: ``cache_db_path`` isolation) unless ``ARXMCP_BM25_INDEX_ROOT``
#: was set explicitly. The shared (non-notebook) path stays ``None``
#: so the global root remains in effect (FM-4 — shared default MUST
#: stay global).
bm25_index_root: Path | None = None
#: BM25 index root, holding one ``v<N>`` directory per corpus version.
#: Unless ``ARXMCP_BM25_INDEX_ROOT`` is set, :meth:`derive_dataset_index_paths`
#: places it in ``derived/bm25`` inside the dataset being served, where
#: a version number cannot collide with another dataset's.
bm25_index_root: Path = legacy_source_path("bm25")

#: SQLite file path for the theorem-names FTS5 index (E10_S02).
#: Sibling to the LanceDB indices under ``var/arxmcp/index/``;
Expand Down Expand Up @@ -703,39 +696,33 @@ def derive_notebook_lancedb_path(self) -> Config:
# and avoids the slug-in-key refactor deferred to m2.
if "cache_db_path" not in self._supplied_path_fields:
self.cache_db_path = derived.parent / "cache" / "retrieval.db"
# notebook-bm25-isolation-m1: redirect the BM25 index root to a
# per-notebook directory unless ARXMCP_BM25_INDEX_ROOT was set
# explicitly. Mirrors the ``cache_db_path`` isolation above.
# ``derived.parent`` = ``var/arxmcp/notebooks/<slug>/`` so the
# BM25 root becomes ``var/arxmcp/notebooks/<slug>/index/bm25``.
# The shared (non-notebook) path stays ``None`` via the
# ``if self.notebook is None: return self`` guard above (FM-4).
if "bm25_index_root" not in self._supplied_path_fields:
self.bm25_index_root = derived.parent / "index" / "bm25"
self._application_paths = replace(
self._application_paths,
lancedb=self.lancedb_path,
retrieval_cache_db=self.cache_db_path,
bm25=self.bm25_index_root or self._application_paths.bm25,
)
return self

@model_validator(mode="after")
def derive_dataset_index_paths(self) -> Config:
"""Place the theorem-name index inside the dataset being served.
"""Place the BM25 and theorem-name indexes inside the dataset served.

Runs after notebook routing has settled :attr:`lancedb_path`, so a
notebook-routed server reads its own notebook's index, and the index
moves with the dataset when cutover or rollback renames it. An
explicit ``ARXMCP_THEOREM_NAMES_DB_PATH`` still wins.
notebook-routed server reads its own notebook's indexes, and they
move with the dataset when cutover or rollback renames it. An
explicit ``ARXMCP_BM25_INDEX_ROOT`` or
``ARXMCP_THEOREM_NAMES_DB_PATH`` still wins.
"""
if "theorem_names_db_path" in self._supplied_path_fields:
return self
self.theorem_names_db_path = derived_index_path(
self.lancedb_path, "theorem_names.db"
)
if "bm25_index_root" not in self._supplied_path_fields:
self.bm25_index_root = derived_index_path(self.lancedb_path, "bm25")
if "theorem_names_db_path" not in self._supplied_path_fields:
self.theorem_names_db_path = derived_index_path(
self.lancedb_path, "theorem_names.db"
)
self._application_paths = replace(
self._application_paths, theorem_names_db=self.theorem_names_db_path
self._application_paths,
bm25=self.bm25_index_root,
theorem_names_db=self.theorem_names_db_path,
)
return self

Expand Down
2 changes: 1 addition & 1 deletion server/paper_metadata_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
durability tier of :mod:`server.cache_sqlite` /
:mod:`server.theorem_names_store`.
- Per-notebook files match the fork-C isolation precedent
(``lancedb/``, ``index/bm25/``, ``cache/retrieval.db``) and keep
(``lancedb/``, ``cache/retrieval.db``) and keep
the CLI-writer / server-reader split off the daemon's live
central DB.
- m1 stays fully additive: no ``NotebooksStore`` schema bump, no
Expand Down
27 changes: 13 additions & 14 deletions server/retrieval/bm25.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

**Architecture: in-memory pickle, NOT LanceDB FTS.** E04_S04 shipped
``bm25.pkl`` + ``chunk_ids.json`` sidecars under
``var/arxmcp/index/bm25/v<N>/``. ``rank_bm25.BM25Okapi.get_scores``
``<lancedb>/derived/bm25/v<N>/``. ``rank_bm25.BM25Okapi.get_scores``
is read-only after construction (pure NumPy ops on indexed
``idf``/``doc_freqs``/``doc_len`` arrays), so no locking is needed
for concurrent reads. The brief's "LanceDB scalar + FTS predicates"
Expand All @@ -17,7 +17,7 @@

**Pickle loader hardening (E04_S04 TODO(E07)).** The original
``ingest/bm25_indexer.py:62-71`` docstring carries a documented-but-
unenforced threat-model: when ``var/arxmcp/index/bm25/`` lives on a
unenforced threat-model: when the BM25 root lives on a
shared filesystem (NFS, Docker bind mount, multi-tenant container),
an attacker who can write to that path achieves RCE in the server
process via ``pickle.load``. This module enforces the mitigation:
Expand Down Expand Up @@ -84,6 +84,7 @@
BM25_INDEX_NAME,
_bm25_version_dir,
build_bm25_index,
resolve_bm25_root,
)
from ingest.tokenizer import tokenize_body

Expand Down Expand Up @@ -144,7 +145,7 @@ class BM25IndexUnsafeError(BM25IndexUnavailableError):
Closes E04_S04 TODO(E07): the loader refuses to ``pickle.load`` a
file that is (a) not owned by the running process's effective
UID, OR (b) world-writable (mode bit ``0o002``). When
``var/arxmcp/index/bm25/`` lives on a shared filesystem an
the BM25 root lives on a shared filesystem an
attacker who can write there achieves RCE in the server process
via crafted pickle data — the stat-check is the
defense-in-depth backstop.
Expand Down Expand Up @@ -400,11 +401,9 @@ async def startup(
the chunks table); optional in tests that don't have a
chunks table to cross-check against.
bm25_index_root: Path | None
notebook-bm25-isolation-m1: optional override for the BM25
artifact root. When ``None`` (default), resolves
:data:`ingest.bm25_indexer.BM25_INDEX_ROOT` at call time
(preserves monkeypatch semantics). Fork-C notebook mode
passes ``config.bm25_index_root`` here.
Optional override for the BM25 artifact root. When ``None``
(default), the index lives in ``derived/bm25`` inside the
dataset at ``lancedb_path``.

Raises
------
Expand Down Expand Up @@ -442,13 +441,13 @@ def _sync_startup(
Parameters
----------
bm25_index_root: Path | None
notebook-bm25-isolation-m1: optional override for the BM25
artifact root. When ``None`` (default), resolves
:data:`ingest.bm25_indexer.BM25_INDEX_ROOT` at call time
(preserves monkeypatch semantics). Fork-C notebook mode
passes ``config.bm25_index_root`` here.
Optional override for the BM25 artifact root. When ``None``
(default), the index lives in ``derived/bm25`` inside the
dataset at ``lancedb_path``.
"""
version_dir = _bm25_version_dir(corpus_version, index_root=bm25_index_root)
version_dir = _bm25_version_dir(
corpus_version, resolve_bm25_root(lancedb_path, bm25_index_root)
)
pkl_path = version_dir / BM25_INDEX_NAME
ids_path = version_dir / BM25_CHUNK_IDS_NAME

Expand Down
28 changes: 0 additions & 28 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -357,34 +357,6 @@ def _patched_bm25_stats_path(tmp_path, monkeypatch):
yield


@pytest.fixture(autouse=True)
def _patched_bm25_index_root(tmp_path, monkeypatch):
"""Redirect ``ingest.bm25_indexer.BM25_INDEX_ROOT`` into ``tmp_path``.

Without this, the BM25 artifact directory is global at
``var/arxmcp/index/bm25/v<N>/`` and stale artifacts from a
previous test run poison subsequent tests — the cross-check
introduced by E07_S01 F4 (``BM25Phase`` cross-checks
``chunk_ids.json`` against the live LanceDB table) trips on a
stale artifact whose ids point to a different test's corpus.

Autouse so every test gets a fresh per-tmp_path artifact root,
matching the discipline of ``_patched_store_stats_path`` and
``_patched_bm25_stats_path``.
"""
try:
import ingest.bm25_indexer as bm25_mod
except ImportError:
yield
return
monkeypatch.setattr(
bm25_mod,
"BM25_INDEX_ROOT",
tmp_path / "bm25_index_root",
)
yield


@pytest.fixture(autouse=True)
def _reset_session_state_for_tests():
"""Drop the per-MCP-session retrieval-cap registry before+after
Expand Down
Loading
Loading