From af01195ffe0d6e99e24eb4a32c8f23f12faac812 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 22:54:22 +0000 Subject: [PATCH] feat(ingest): keep BM25 index in its dataset BM25 indexes lived under index/bm25/v for the shared corpus and notebooks//index/bm25/v per notebook. A corpus_version is only unique within its dataset, so a root shared across datasets could serve one dataset's index for another's version, and a notebook cutover swapped the chunks while leaving their BM25 index behind. The root is now /derived/bm25 for every dataset. Config and ApplicationPaths derive it from the dataset unless ARXMCP_BM25_INDEX_ROOT is set; build_bm25_index and the server's auto-build default to it through resolve_bm25_root. The global BM25_INDEX_ROOT, and the test hook that patched it, are gone: each test dataset now carries its own index. Upgrade: indexes under the old roots are no longer read. The server rebuilds the missing version on its next start. Closes #554 --- ingest/bm25_indexer.py | 55 ++++++++-------- server/application_paths.py | 11 ++-- server/config.py | 51 ++++++--------- server/paper_metadata_store.py | 2 +- server/retrieval/bm25.py | 27 ++++---- tests/conftest.py | 28 -------- tests/retrieval/test_bm25.py | 52 ++++++--------- tests/test_application_paths.py | 4 +- tests/test_bm25.py | 97 +++++++++++----------------- tests/test_notebook_cutover.py | 31 ++------- tests/test_server_startup.py | 42 +++--------- tests/tools/test_notebook_scripts.py | 9 ++- tools/notebook_cutover.py | 20 ++---- tools/notebook_ingest.py | 32 ++------- 14 files changed, 159 insertions(+), 302 deletions(-) diff --git a/ingest/bm25_indexer.py b/ingest/bm25_indexer.py index 9f881890..8bf1e1b3 100644 --- a/ingest/bm25_indexer.py +++ b/ingest/bm25_indexer.py @@ -7,7 +7,7 @@ ``body_tokens`` from the pinned LanceDB dataset version (E04_S02 :func:`server.corpus.open_chunks_table`) and feeding the whitespace-split token streams to :class:`rank_bm25.BM25Okapi`. The -fitted index is pickled to ``var/arxmcp/index/bm25/v/bm25.pkl`` and +fitted index is pickled to ``/derived/bm25/v/bm25.pkl`` and the row-aligned chunk_id list is written as ``chunk_ids.json`` next to it. The MCP server (E07, Sonnet B) loads the pair on first lexical query; it never enters the in-memory ingestion path. @@ -54,7 +54,7 @@ **Pickle security.** ``bm25.pkl`` is produced locally from trusted LanceDB data by this module. ``pickle.load`` on an untrusted file is remote-code-execution; the path -``var/arxmcp/index/bm25/`` MUST be treated as trusted-local. Threat 6 +the BM25 root MUST be treated as trusted-local. Threat 6 (``08-security-observability-ops.md``) covers model-weight pickles (deny); the BM25 pickle is application data with the analogous- narrower attack surface (trust local, deny remote). @@ -63,7 +63,7 @@ verify file ownership matches process UID and refuse world-writable paths before calling ``pickle.load``. The defense-in-depth concern (M1 from the E04_S04 critique) is that documentation alone is not -enforcement. Rationale: when ``var/arxmcp/index/bm25/`` lives on a +enforcement. Rationale: when the BM25 root lives on a shared filesystem (NFS, Docker bind mount, multi-tenant container) an attacker who can write to that path achieves RCE in the server process. The mitigation is a simple stat-based check at load time; @@ -95,39 +95,40 @@ REPO_ROOT = Path(__file__).resolve().parent.parent -# The directory layout convention: ``var/arxmcp/index/bm25/v/``. +# The directory layout convention: ``/derived/bm25/v/``. # Confine the ``f"v{N}"`` literal to ``_bm25_version_dir`` below — every # other reference goes through that helper. BM25_DIR_NAME = "bm25" BM25_INDEX_NAME = "bm25.pkl" BM25_CHUNK_IDS_NAME = "chunk_ids.json" -BM25_INDEX_ROOT = REPO_ROOT / "var" / "arxmcp" / "index" / BM25_DIR_NAME BM25_STATS_PATH = REPO_ROOT / "var" / "arxmcp" / "ops" / "bm25-stats.jsonl" -def _bm25_version_dir( - corpus_version: int, - index_root: Path | None = None, +def resolve_bm25_root( + lancedb_path: str | Path, index_root: str | Path | None = None, ) -> Path: - """Return the per-version BM25 index directory. + """The BM25 root for the dataset at ``lancedb_path``. + + ``index_root`` when given, else ``derived/bm25`` inside the dataset. + A ``corpus_version`` is unique only within its dataset, so keeping the + ``v`` directories there means two datasets at the same version + never share an index, and cutover or rollback moves the index with + the chunks it was built from. + """ + if index_root is not None: + return Path(index_root) + from server.application_paths import derived_index_path + + return derived_index_path(lancedb_path, BM25_DIR_NAME) + + +def _bm25_version_dir(corpus_version: int, index_root: Path) -> Path: + """Return the per-version BM25 index directory under ``index_root``. Confines the ``f"v{N}"`` literal to a single function so a future rename (e.g. zero-padding to ``v00007``) is a one-line change. - - Parameters - ---------- - corpus_version: - MVCC version integer, e.g. 1. - index_root: - Optional override for the BM25 artifact root. When ``None`` - (the default), resolves :data:`BM25_INDEX_ROOT` **at call time** - so ``monkeypatch.setattr(bm25_mod, "BM25_INDEX_ROOT", ...)`` - continues to intercept the path in tests (notebook-bm25-isolation-m1). - When provided (fork-C notebook mode), uses that path as the root - so the per-notebook index is isolated from the shared corpus. """ - root = index_root if index_root is not None else BM25_INDEX_ROOT - return root / f"v{corpus_version}" + return Path(index_root) / f"v{corpus_version}" # --------------------------------------------------------------------------- @@ -270,7 +271,7 @@ def build_bm25_index( fits a :class:`rank_bm25.BM25Okapi` over the whitespace-split token streams, and writes ``bm25.pkl`` + ``chunk_ids.json`` to ``/v/`` atomically (where ``index_root`` - defaults to :data:`BM25_INDEX_ROOT` when ``None``). + defaults to ``derived/bm25`` inside the dataset when ``None``). .. warning:: @@ -324,7 +325,9 @@ def build_bm25_index( ) start = time.monotonic() - version_dir = _bm25_version_dir(corpus_version, index_root=index_root) + version_dir = _bm25_version_dir( + corpus_version, resolve_bm25_root(lancedb_path, index_root) + ) pkl_path = version_dir / BM25_INDEX_NAME ids_path = version_dir / BM25_CHUNK_IDS_NAME @@ -452,8 +455,8 @@ def build_bm25_index( "BM25_CHUNK_IDS_NAME", "BM25_DIR_NAME", "BM25_INDEX_NAME", - "BM25_INDEX_ROOT", "BM25_STATS_PATH", "BM25Stats", "build_bm25_index", + "resolve_bm25_root", ] diff --git a/server/application_paths.py b/server/application_paths.py index b74bab3e..77ddfee0 100644 --- a/server/application_paths.py +++ b/server/application_paths.py @@ -24,7 +24,7 @@ _LAYOUT = ( ("corpus", "corpus"), ("index", "index"), ("lancedb", "index/lancedb"), ("kuzu", "index/kuzu"), - ("bm25", "index/bm25"), ("sqlite", "index/sqlite"), + ("bm25", "index/lancedb/derived/bm25"), ("sqlite", "index/sqlite"), ("theorem_names_db", "index/lancedb/derived/theorem_names.db"), ("notebooks", "notebooks"), ("cache", "cache"), ("retrieval_cache_db", "cache/retrieval.db"), @@ -187,11 +187,10 @@ def resolve( values[field] = resolved else: values[field] = _inside(canonical_root, path, field) - if not supplied.get(alias_by_field["theorem_names_db"]): - # Follows the dataset, including a dataset moved by its own alias. - values["theorem_names_db"] = derived_index_path( - values["lancedb"], "theorem_names.db" - ) + # Derived indexes follow the dataset, including one moved by its alias. + for field, name in (("bm25", "bm25"), ("theorem_names_db", "theorem_names.db")): + if not supplied.get(alias_by_field[field]): + values[field] = derived_index_path(values["lancedb"], name) return cls( selected, canonical_root, diff --git a/server/config.py b/server/config.py index 2b700549..ee410405 100644 --- a/server/config.py +++ b/server/config.py @@ -149,18 +149,11 @@ class Config(BaseSettings): #: directory is created at ``Resources.startup()`` time. cache_db_path: Path = legacy_source_path("retrieval_cache_db") - #: Per-notebook BM25 index root (notebook-bm25-isolation-m1). - #: Default ``None`` means "resolve :data:`ingest.bm25_indexer.BM25_INDEX_ROOT` - #: at call time" — preserving the conftest autouse - #: ``monkeypatch.setattr(bm25_mod, "BM25_INDEX_ROOT", ...)`` used by - #: ~40 tests. When :attr:`notebook` is set, the - #: :meth:`derive_notebook_lancedb_path` validator rewrites this to - #: ``var/arxmcp/notebooks//index/bm25`` (mirroring the - #: ``cache_db_path`` isolation) unless ``ARXMCP_BM25_INDEX_ROOT`` - #: was set explicitly. The shared (non-notebook) path stays ``None`` - #: so the global root remains in effect (FM-4 — shared default MUST - #: stay global). - bm25_index_root: Path | None = None + #: BM25 index root, holding one ``v`` directory per corpus version. + #: Unless ``ARXMCP_BM25_INDEX_ROOT`` is set, :meth:`derive_dataset_index_paths` + #: places it in ``derived/bm25`` inside the dataset being served, where + #: a version number cannot collide with another dataset's. + bm25_index_root: Path = legacy_source_path("bm25") #: SQLite file path for the theorem-names FTS5 index (E10_S02). #: Sibling to the LanceDB indices under ``var/arxmcp/index/``; @@ -703,39 +696,33 @@ def derive_notebook_lancedb_path(self) -> Config: # and avoids the slug-in-key refactor deferred to m2. if "cache_db_path" not in self._supplied_path_fields: self.cache_db_path = derived.parent / "cache" / "retrieval.db" - # notebook-bm25-isolation-m1: redirect the BM25 index root to a - # per-notebook directory unless ARXMCP_BM25_INDEX_ROOT was set - # explicitly. Mirrors the ``cache_db_path`` isolation above. - # ``derived.parent`` = ``var/arxmcp/notebooks//`` so the - # BM25 root becomes ``var/arxmcp/notebooks//index/bm25``. - # The shared (non-notebook) path stays ``None`` via the - # ``if self.notebook is None: return self`` guard above (FM-4). - if "bm25_index_root" not in self._supplied_path_fields: - self.bm25_index_root = derived.parent / "index" / "bm25" self._application_paths = replace( self._application_paths, lancedb=self.lancedb_path, retrieval_cache_db=self.cache_db_path, - bm25=self.bm25_index_root or self._application_paths.bm25, ) return self @model_validator(mode="after") def derive_dataset_index_paths(self) -> Config: - """Place the theorem-name index inside the dataset being served. + """Place the BM25 and theorem-name indexes inside the dataset served. Runs after notebook routing has settled :attr:`lancedb_path`, so a - notebook-routed server reads its own notebook's index, and the index - moves with the dataset when cutover or rollback renames it. An - explicit ``ARXMCP_THEOREM_NAMES_DB_PATH`` still wins. + notebook-routed server reads its own notebook's indexes, and they + move with the dataset when cutover or rollback renames it. An + explicit ``ARXMCP_BM25_INDEX_ROOT`` or + ``ARXMCP_THEOREM_NAMES_DB_PATH`` still wins. """ - if "theorem_names_db_path" in self._supplied_path_fields: - return self - self.theorem_names_db_path = derived_index_path( - self.lancedb_path, "theorem_names.db" - ) + if "bm25_index_root" not in self._supplied_path_fields: + self.bm25_index_root = derived_index_path(self.lancedb_path, "bm25") + if "theorem_names_db_path" not in self._supplied_path_fields: + self.theorem_names_db_path = derived_index_path( + self.lancedb_path, "theorem_names.db" + ) self._application_paths = replace( - self._application_paths, theorem_names_db=self.theorem_names_db_path + self._application_paths, + bm25=self.bm25_index_root, + theorem_names_db=self.theorem_names_db_path, ) return self diff --git a/server/paper_metadata_store.py b/server/paper_metadata_store.py index c393392c..e2ca4f63 100644 --- a/server/paper_metadata_store.py +++ b/server/paper_metadata_store.py @@ -18,7 +18,7 @@ durability tier of :mod:`server.cache_sqlite` / :mod:`server.theorem_names_store`. - Per-notebook files match the fork-C isolation precedent - (``lancedb/``, ``index/bm25/``, ``cache/retrieval.db``) and keep + (``lancedb/``, ``cache/retrieval.db``) and keep the CLI-writer / server-reader split off the daemon's live central DB. - m1 stays fully additive: no ``NotebooksStore`` schema bump, no diff --git a/server/retrieval/bm25.py b/server/retrieval/bm25.py index d0cd21c0..431c8a8b 100644 --- a/server/retrieval/bm25.py +++ b/server/retrieval/bm25.py @@ -8,7 +8,7 @@ **Architecture: in-memory pickle, NOT LanceDB FTS.** E04_S04 shipped ``bm25.pkl`` + ``chunk_ids.json`` sidecars under -``var/arxmcp/index/bm25/v/``. ``rank_bm25.BM25Okapi.get_scores`` +``/derived/bm25/v/``. ``rank_bm25.BM25Okapi.get_scores`` is read-only after construction (pure NumPy ops on indexed ``idf``/``doc_freqs``/``doc_len`` arrays), so no locking is needed for concurrent reads. The brief's "LanceDB scalar + FTS predicates" @@ -17,7 +17,7 @@ **Pickle loader hardening (E04_S04 TODO(E07)).** The original ``ingest/bm25_indexer.py:62-71`` docstring carries a documented-but- -unenforced threat-model: when ``var/arxmcp/index/bm25/`` lives on a +unenforced threat-model: when the BM25 root lives on a shared filesystem (NFS, Docker bind mount, multi-tenant container), an attacker who can write to that path achieves RCE in the server process via ``pickle.load``. This module enforces the mitigation: @@ -84,6 +84,7 @@ BM25_INDEX_NAME, _bm25_version_dir, build_bm25_index, + resolve_bm25_root, ) from ingest.tokenizer import tokenize_body @@ -144,7 +145,7 @@ class BM25IndexUnsafeError(BM25IndexUnavailableError): Closes E04_S04 TODO(E07): the loader refuses to ``pickle.load`` a file that is (a) not owned by the running process's effective UID, OR (b) world-writable (mode bit ``0o002``). When - ``var/arxmcp/index/bm25/`` lives on a shared filesystem an + the BM25 root lives on a shared filesystem an attacker who can write there achieves RCE in the server process via crafted pickle data — the stat-check is the defense-in-depth backstop. @@ -400,11 +401,9 @@ async def startup( the chunks table); optional in tests that don't have a chunks table to cross-check against. bm25_index_root: Path | None - notebook-bm25-isolation-m1: optional override for the BM25 - artifact root. When ``None`` (default), resolves - :data:`ingest.bm25_indexer.BM25_INDEX_ROOT` at call time - (preserves monkeypatch semantics). Fork-C notebook mode - passes ``config.bm25_index_root`` here. + Optional override for the BM25 artifact root. When ``None`` + (default), the index lives in ``derived/bm25`` inside the + dataset at ``lancedb_path``. Raises ------ @@ -442,13 +441,13 @@ def _sync_startup( Parameters ---------- bm25_index_root: Path | None - notebook-bm25-isolation-m1: optional override for the BM25 - artifact root. When ``None`` (default), resolves - :data:`ingest.bm25_indexer.BM25_INDEX_ROOT` at call time - (preserves monkeypatch semantics). Fork-C notebook mode - passes ``config.bm25_index_root`` here. + Optional override for the BM25 artifact root. When ``None`` + (default), the index lives in ``derived/bm25`` inside the + dataset at ``lancedb_path``. """ - version_dir = _bm25_version_dir(corpus_version, index_root=bm25_index_root) + version_dir = _bm25_version_dir( + corpus_version, resolve_bm25_root(lancedb_path, bm25_index_root) + ) pkl_path = version_dir / BM25_INDEX_NAME ids_path = version_dir / BM25_CHUNK_IDS_NAME diff --git a/tests/conftest.py b/tests/conftest.py index 7b91c677..eb23ac0c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -357,34 +357,6 @@ def _patched_bm25_stats_path(tmp_path, monkeypatch): yield -@pytest.fixture(autouse=True) -def _patched_bm25_index_root(tmp_path, monkeypatch): - """Redirect ``ingest.bm25_indexer.BM25_INDEX_ROOT`` into ``tmp_path``. - - Without this, the BM25 artifact directory is global at - ``var/arxmcp/index/bm25/v/`` and stale artifacts from a - previous test run poison subsequent tests — the cross-check - introduced by E07_S01 F4 (``BM25Phase`` cross-checks - ``chunk_ids.json`` against the live LanceDB table) trips on a - stale artifact whose ids point to a different test's corpus. - - Autouse so every test gets a fresh per-tmp_path artifact root, - matching the discipline of ``_patched_store_stats_path`` and - ``_patched_bm25_stats_path``. - """ - try: - import ingest.bm25_indexer as bm25_mod - except ImportError: - yield - return - monkeypatch.setattr( - bm25_mod, - "BM25_INDEX_ROOT", - tmp_path / "bm25_index_root", - ) - yield - - @pytest.fixture(autouse=True) def _reset_session_state_for_tests(): """Drop the per-MCP-session retrieval-cap registry before+after diff --git a/tests/retrieval/test_bm25.py b/tests/retrieval/test_bm25.py index c59dbd29..fd868d77 100644 --- a/tests/retrieval/test_bm25.py +++ b/tests/retrieval/test_bm25.py @@ -36,6 +36,7 @@ _atomic_write_bytes, _bm25_version_dir, build_bm25_index, + resolve_bm25_root, ) from ingest.chunker_types import CHUNKER_VERSION, ChunkRecord from ingest.embedder import EMBEDDER_VERSION, EMBEDDING_DIM @@ -165,17 +166,6 @@ def _embeddings_for(chunks: list[ChunkRecord], *, seed: int = 0) -> EmbedRecord: ) -@pytest.fixture -def _bm25_root(tmp_path, monkeypatch) -> Path: - """Redirect ``BM25_INDEX_ROOT`` into tmp_path so the test does - not pollute the developer's checkout-local ``var/arxmcp/``.""" - import ingest.bm25_indexer as bm25_mod - - root = tmp_path / "bm25_root" - monkeypatch.setattr(bm25_mod, "BM25_INDEX_ROOT", root) - return root - - @pytest.fixture def _seeded_lancedb(tmp_path) -> tuple[Path, int]: """Write the curated corpus to a fresh LanceDB and return @@ -188,7 +178,7 @@ def _seeded_lancedb(tmp_path) -> tuple[Path, int]: @pytest.fixture -def _bm25_phase(_seeded_lancedb, _bm25_root) -> BM25Phase: +def _bm25_phase(_seeded_lancedb) -> BM25Phase: """Build the BM25 artifact + return a loaded ``BM25Phase``. Synchronous helper — calls ``BM25Phase._sync_startup`` directly so @@ -471,12 +461,12 @@ class TestAutoBuild: """``BM25Phase.startup`` auto-builds the artifact if missing.""" def test_auto_build_when_artifact_missing( - self, _seeded_lancedb, _bm25_root + self, _seeded_lancedb ): """Fresh LanceDB + no pre-existing BM25 artifact: startup builds it and returns a working BM25Phase.""" lancedb_path, version = _seeded_lancedb - version_dir = _bm25_version_dir(version) + version_dir = _bm25_version_dir(version, resolve_bm25_root(lancedb_path)) # Sanity: artifact does NOT exist yet. assert not (version_dir / BM25_INDEX_NAME).is_file() @@ -487,14 +477,14 @@ def test_auto_build_when_artifact_missing( assert (version_dir / BM25_CHUNK_IDS_NAME).is_file() def test_warm_start_uses_existing_artifact( - self, _seeded_lancedb, _bm25_root + self, _seeded_lancedb ): """Pre-build the artifact, then call startup. The build is idempotent-skip; startup loads the existing pickle.""" lancedb_path, version = _seeded_lancedb # Pre-build. build_bm25_index(lancedb_path, corpus_version=version) - version_dir = _bm25_version_dir(version) + version_dir = _bm25_version_dir(version, resolve_bm25_root(lancedb_path)) pkl_mtime_before = (version_dir / BM25_INDEX_NAME).stat().st_mtime phase = BM25Phase._sync_startup(lancedb_path, version) @@ -503,7 +493,7 @@ def test_warm_start_uses_existing_artifact( pkl_mtime_after = (version_dir / BM25_INDEX_NAME).stat().st_mtime assert pkl_mtime_before == pkl_mtime_after - def test_build_failure_raises_unavailable(self, tmp_path, _bm25_root): + def test_build_failure_raises_unavailable(self, tmp_path): """If LanceDB is missing entirely, build raises ValueError / FileNotFoundError, which startup wraps as ``BM25IndexUnavailableError``.""" @@ -556,13 +546,13 @@ class TestArtifactIntegrity: A misaligned pair would silently return wrong chunk_ids.""" def test_misaligned_corpus_size_rejected( - self, _seeded_lancedb, _bm25_root + self, _seeded_lancedb ): """Build a normal index, then truncate chunk_ids.json so it has fewer entries than the pickle. Startup must raise.""" lancedb_path, version = _seeded_lancedb build_bm25_index(lancedb_path, corpus_version=version) - version_dir = _bm25_version_dir(version) + version_dir = _bm25_version_dir(version, resolve_bm25_root(lancedb_path)) ids_path = version_dir / BM25_CHUNK_IDS_NAME # Truncate chunk_ids to half its size. @@ -649,7 +639,7 @@ class TestAsyncStartup: """``BM25Phase.startup`` is the production async entry point.""" def test_async_startup_returns_phase( - self, _seeded_lancedb, _bm25_root + self, _seeded_lancedb ): lancedb_path, version = _seeded_lancedb phase = asyncio.run(BM25Phase.startup(lancedb_path, version)) @@ -664,13 +654,13 @@ def test_async_startup_returns_phase( class TestCorruptPickleHandling: - def test_corrupt_pickle_raises(self, _seeded_lancedb, _bm25_root): + def test_corrupt_pickle_raises(self, _seeded_lancedb): """F8 fix: narrowed pytest.raises tuple. A corrupted bm25.pkl must raise one of the typed pickle/io errors — NOT a bare Exception that would mask any unrelated bug as 'success'.""" lancedb_path, version = _seeded_lancedb build_bm25_index(lancedb_path, corpus_version=version) - version_dir = _bm25_version_dir(version) + version_dir = _bm25_version_dir(version, resolve_bm25_root(lancedb_path)) pkl_path = version_dir / BM25_INDEX_NAME # Have to rewrite via _atomic_write_bytes so the F3 chmod # 0o600 is applied (otherwise the file-safety check would @@ -697,13 +687,13 @@ class TestParentDirectorySafety: write to the dir can swap files regardless of file mode.""" def test_world_writable_parent_dir_rejected( - self, _seeded_lancedb, _bm25_root + self, _seeded_lancedb ): from server.retrieval.bm25 import _assert_dir_safe lancedb_path, version = _seeded_lancedb build_bm25_index(lancedb_path, corpus_version=version) - version_dir = _bm25_version_dir(version) + version_dir = _bm25_version_dir(version, resolve_bm25_root(lancedb_path)) original_mode = version_dir.stat().st_mode try: @@ -732,11 +722,11 @@ class TestChunkIdsLiveCrossCheck: must trigger BM25IndexUnavailableError at startup, not produce phantom candidates.""" - def test_phantom_chunk_id_rejected(self, _seeded_lancedb, _bm25_root): + def test_phantom_chunk_id_rejected(self, _seeded_lancedb): lancedb_path, version = _seeded_lancedb # Build, then tamper chunk_ids.json to include a phantom id. build_bm25_index(lancedb_path, corpus_version=version) - version_dir = _bm25_version_dir(version) + version_dir = _bm25_version_dir(version, resolve_bm25_root(lancedb_path)) ids_path = version_dir / BM25_CHUNK_IDS_NAME original = json.loads(ids_path.read_text()) # Replace one id with a non-existent phantom (same length so @@ -763,7 +753,7 @@ def test_phantom_chunk_id_rejected(self, _seeded_lancedb, _bm25_root): ) def test_no_cross_check_when_live_chunk_ids_none( - self, _seeded_lancedb, _bm25_root + self, _seeded_lancedb ): """When live_chunk_ids is None (the test-friendly default), no cross-check fires — preserves the existing test surface.""" @@ -820,11 +810,11 @@ def test_text_write_under_open_umask_is_0o600(self, tmp_path): class TestArtifactIntegrityExtended: """F7 fix: cover both directions of length mismatch + duplicates.""" - def test_extra_chunk_ids_rejected(self, _seeded_lancedb, _bm25_root): + def test_extra_chunk_ids_rejected(self, _seeded_lancedb): """chunk_ids.json longer than corpus_size must also raise.""" lancedb_path, version = _seeded_lancedb build_bm25_index(lancedb_path, corpus_version=version) - version_dir = _bm25_version_dir(version) + version_dir = _bm25_version_dir(version, resolve_bm25_root(lancedb_path)) ids_path = version_dir / BM25_CHUNK_IDS_NAME from ingest.bm25_indexer import _atomic_write_text @@ -848,7 +838,7 @@ class TestEmptyCorpusFailure: """F9 fix: a corpus with zero non-empty body_tokens fails the auto-build path, which must surface as BM25IndexUnavailableError.""" - def test_empty_body_tokens_corpus_raises(self, tmp_path, _bm25_root): + def test_empty_body_tokens_corpus_raises(self, tmp_path): """Build a LanceDB corpus where every chunk has empty body_tokens. ``build_bm25_index`` raises ValueError; the startup wrapper translates to BM25IndexUnavailableError.""" @@ -931,7 +921,7 @@ class TestResourcesIntegration: a test. This class probes the integration end-to-end.""" def test_resources_startup_populates_bm25_phase( - self, _seeded_lancedb, _bm25_root + self, _seeded_lancedb ): """``Resources.startup(cfg)`` MUST populate ``r.bm25_phase`` with a working ``BM25Phase`` instance.""" diff --git a/tests/test_application_paths.py b/tests/test_application_paths.py index 9ed9b67a..4eaeea83 100644 --- a/tests/test_application_paths.py +++ b/tests/test_application_paths.py @@ -80,7 +80,7 @@ def test_installed_config_uses_one_canonical_root( assert config.data_dir == root.resolve() assert config.lancedb_path == root / "index/lancedb" assert config.kuzu_path == root / "index/kuzu" - assert config.bm25_index_root == root / "index/bm25" + assert config.bm25_index_root == root / "index/lancedb/derived/bm25" assert config.notebooks_db_path == root / "cache/notebooks.db" @@ -100,7 +100,7 @@ def test_installed_notebook_uses_derived_paths( config = Config() assert config.lancedb_path == notebook / "lancedb" assert config.cache_db_path == notebook / "cache/retrieval.db" - assert config.bm25_index_root == notebook / "index/bm25" + assert config.bm25_index_root == notebook / "lancedb/derived/bm25" assert config.application_paths.lancedb == config.lancedb_path assert config.application_paths.retrieval_cache_db == config.cache_db_path assert config.application_paths.bm25 == config.bm25_index_root diff --git a/tests/test_bm25.py b/tests/test_bm25.py index 84557392..37cbc493 100644 --- a/tests/test_bm25.py +++ b/tests/test_bm25.py @@ -187,14 +187,9 @@ def test_docstring_threat1_deferral(self): class TestBuildIndex: def test_writes_pkl_and_chunk_ids(self, tmp_path, monkeypatch): """AC: bm25.pkl + chunk_ids.json are written under - ``var/arxmcp/index/bm25/v/``.""" - # Redirect BM25_INDEX_ROOT into tmp_path so we don't pollute - # the developer's var/ tree. + ``derived/bm25/v/`` inside the dataset.""" import ingest.bm25_indexer as bm25_mod - monkeypatch.setattr( - bm25_mod, "BM25_INDEX_ROOT", tmp_path / "bm25" - ) chunks = _curated_corpus() embeddings = _embeddings_for(chunks, seed=1) @@ -203,7 +198,9 @@ def test_writes_pkl_and_chunk_ids(self, tmp_path, monkeypatch): ) build_bm25_index(tmp_path / "lancedb", corpus_version=version) - version_dir = bm25_mod._bm25_version_dir(version) + version_dir = bm25_mod._bm25_version_dir( + version, bm25_mod.resolve_bm25_root(tmp_path / "lancedb") + ) assert (version_dir / BM25_INDEX_NAME).is_file() assert (version_dir / BM25_CHUNK_IDS_NAME).is_file() @@ -215,9 +212,6 @@ def test_chunk_ids_json_is_aligned_with_bm25_corpus( equal-length sequences.""" import ingest.bm25_indexer as bm25_mod - monkeypatch.setattr( - bm25_mod, "BM25_INDEX_ROOT", tmp_path / "bm25" - ) chunks = _curated_corpus() embeddings = _embeddings_for(chunks, seed=2) @@ -226,7 +220,9 @@ def test_chunk_ids_json_is_aligned_with_bm25_corpus( ) build_bm25_index(tmp_path / "lancedb", corpus_version=version) - version_dir = bm25_mod._bm25_version_dir(version) + version_dir = bm25_mod._bm25_version_dir( + version, bm25_mod.resolve_bm25_root(tmp_path / "lancedb") + ) with (version_dir / BM25_INDEX_NAME).open("rb") as fh: bm25 = pickle.load(fh) chunk_ids = json.loads( @@ -250,9 +246,6 @@ def test_query_spec_mathrm_pic_returns_target_chunk( containing those tokens with the highest score.""" import ingest.bm25_indexer as bm25_mod - monkeypatch.setattr( - bm25_mod, "BM25_INDEX_ROOT", tmp_path / "bm25" - ) chunks = _curated_corpus() embeddings = _embeddings_for(chunks, seed=3) @@ -262,7 +255,9 @@ def test_query_spec_mathrm_pic_returns_target_chunk( ) build_bm25_index(tmp_path / "lancedb", corpus_version=version) - version_dir = bm25_mod._bm25_version_dir(version) + version_dir = bm25_mod._bm25_version_dir( + version, bm25_mod.resolve_bm25_root(tmp_path / "lancedb") + ) with (version_dir / BM25_INDEX_NAME).open("rb") as fh: bm25 = pickle.load(fh) chunk_ids = json.loads( @@ -300,9 +295,6 @@ def test_query_disjoint_vocabulary_does_not_match_target( """ import ingest.bm25_indexer as bm25_mod - monkeypatch.setattr( - bm25_mod, "BM25_INDEX_ROOT", tmp_path / "bm25" - ) chunks = _curated_corpus() embeddings = _embeddings_for(chunks, seed=4) @@ -313,7 +305,9 @@ def test_query_disjoint_vocabulary_does_not_match_target( ) build_bm25_index(tmp_path / "lancedb", corpus_version=version) - version_dir = bm25_mod._bm25_version_dir(version) + version_dir = bm25_mod._bm25_version_dir( + version, bm25_mod.resolve_bm25_root(tmp_path / "lancedb") + ) with (version_dir / BM25_INDEX_NAME).open("rb") as fh: bm25 = pickle.load(fh) chunk_ids = json.loads( @@ -341,9 +335,6 @@ class TestIdempotency: def test_rerun_is_skipped_when_files_exist(self, tmp_path, monkeypatch): import ingest.bm25_indexer as bm25_mod - monkeypatch.setattr( - bm25_mod, "BM25_INDEX_ROOT", tmp_path / "bm25" - ) chunks = _curated_corpus() embeddings = _embeddings_for(chunks, seed=5) @@ -352,7 +343,9 @@ def test_rerun_is_skipped_when_files_exist(self, tmp_path, monkeypatch): ) build_bm25_index(tmp_path / "lancedb", corpus_version=version) - version_dir = bm25_mod._bm25_version_dir(version) + version_dir = bm25_mod._bm25_version_dir( + version, bm25_mod.resolve_bm25_root(tmp_path / "lancedb") + ) pkl_path = version_dir / BM25_INDEX_NAME ids_path = version_dir / BM25_CHUNK_IDS_NAME # Capture mtimes; a no-op re-run leaves them untouched. @@ -380,9 +373,6 @@ def test_partial_state_triggers_rebuild(self, tmp_path, monkeypatch): """ import ingest.bm25_indexer as bm25_mod - monkeypatch.setattr( - bm25_mod, "BM25_INDEX_ROOT", tmp_path / "bm25" - ) chunks = _curated_corpus() embeddings = _embeddings_for(chunks, seed=6) @@ -391,7 +381,9 @@ def test_partial_state_triggers_rebuild(self, tmp_path, monkeypatch): ) build_bm25_index(tmp_path / "lancedb", corpus_version=version) - version_dir = bm25_mod._bm25_version_dir(version) + version_dir = bm25_mod._bm25_version_dir( + version, bm25_mod.resolve_bm25_root(tmp_path / "lancedb") + ) pkl_path = version_dir / BM25_INDEX_NAME ids_path = version_dir / BM25_CHUNK_IDS_NAME @@ -419,12 +411,6 @@ def test_empty_corpus_raises_value_error(self, tmp_path, monkeypatch): """An empty BM25 corpus produces NaN IDFs (rank_bm25). Surface explicitly as a ValueError rather than writing a useless index.""" - import ingest.bm25_indexer as bm25_mod - - monkeypatch.setattr( - bm25_mod, "BM25_INDEX_ROOT", tmp_path / "bm25" - ) - # Build an empty LanceDB table by writing zero chunks. embeddings = EmbedRecord( chunk_ids_stmt=[], @@ -534,9 +520,6 @@ class TestStatsLogging: def test_stats_line_appended_on_build(self, tmp_path, monkeypatch): import ingest.bm25_indexer as bm25_mod - monkeypatch.setattr( - bm25_mod, "BM25_INDEX_ROOT", tmp_path / "bm25" - ) chunks = _curated_corpus() embeddings = _embeddings_for(chunks, seed=7) @@ -565,9 +548,6 @@ def test_stats_line_records_skipped_on_no_op_rerun( ): import ingest.bm25_indexer as bm25_mod - monkeypatch.setattr( - bm25_mod, "BM25_INDEX_ROOT", tmp_path / "bm25" - ) chunks = _curated_corpus() embeddings = _embeddings_for(chunks, seed=8) @@ -678,10 +658,7 @@ def test_to_dict_alphabetical_keys(self): class TestBM25IndexRootIsolation: """Regression: per-notebook BM25 index root isolation. - These tests MUST FAIL on the pre-fix code (before - ``notebook-bm25-isolation-m1``) where ``_bm25_version_dir`` always - used the global ``BM25_INDEX_ROOT`` and ``build_bm25_index`` had no - ``index_root`` parameter. + These tests pin that an index lives under the root it was built for. Post-fix they pass because the ``index_root`` parameter makes each call use a different directory tree. @@ -779,25 +756,27 @@ def test_build_bm25_index_per_root_no_overlap(self, tmp_path): f"root_b chunk_ids should contain notebook-B chunks; got {ids_list_b}" ) - def test_none_index_root_uses_monkeypatched_global(self, tmp_path, monkeypatch): - """FM-6: when ``index_root=None`` (the default), ``_bm25_version_dir`` - reads ``BM25_INDEX_ROOT`` at call time, so the conftest autouse - monkeypatch continues to intercept the path in tests. + def test_datasets_at_the_same_version_keep_separate_indexes(self, tmp_path): + """With no explicit root, each dataset holds its own index, so two + datasets that both reach version N never read each other's.""" + import json as _json - This asserts the lazy-resolution semantic that keeps ~40 existing - tests green after the fix. - """ import ingest.bm25_indexer as bm25_mod - patched_root = tmp_path / "patched_root" - monkeypatch.setattr(bm25_mod, "BM25_INDEX_ROOT", patched_root) - - version = 3 - result = bm25_mod._bm25_version_dir(version, index_root=None) - assert result == patched_root / f"v{version}", ( - f"index_root=None must resolve to the (patchable) " - f"BM25_INDEX_ROOT at call time; got {result}" - ) + ids = {} + for name, paper_id, chunk_hash in ( + ("lancedb_a", "2301.00001", "aaaa0001"), + ("lancedb_b", "2302.00002", "bbbb0002"), + ): + chunks = [_curated_chunk(paper_id, chunk_hash, body_tokens="alpha lemma")] + version = write_chunks(chunks, _embeddings_for(chunks, seed=1), tmp_path / name) + bm25_mod.build_bm25_index(tmp_path / name, corpus_version=version) + version_dir = tmp_path / name / "derived" / "bm25" / f"v{version}" + ids[name] = (version, _json.loads((version_dir / "chunk_ids.json").read_text())) + + assert ids["lancedb_a"][0] == ids["lancedb_b"][0] + assert all("2301" in cid for cid in ids["lancedb_a"][1]) + assert all("2302" in cid for cid in ids["lancedb_b"][1]) def test_idempotent_skip_is_scoped_per_root(self, tmp_path, monkeypatch): """F3: the idempotent rebuild-skip keys on the PER-NOTEBOOK diff --git a/tests/test_notebook_cutover.py b/tests/test_notebook_cutover.py index eda295f6..51816433 100644 --- a/tests/test_notebook_cutover.py +++ b/tests/test_notebook_cutover.py @@ -91,27 +91,6 @@ def test_cutover_swaps_and_backs_up(nb_base) -> None: assert (backups[0] / "_sentinel.txt").read_text() == "v369" -def test_cutover_builds_no_global_bm25(nb_base, monkeypatch, tmp_path) -> None: - """F1 rectification: the cutover must NOT write into the GLOBAL BM25 - namespace (var/arxmcp/index/bm25/v/) — that root collides across - notebooks + the shared corpus on per-dataset MVCC versions. Point the BM25 - root at a fresh tmp dir and assert the cutover leaves it empty (the build - was removed; fork-C startup auto-builds, fork-A is dense-only).""" - import ingest.bm25_indexer as bm25_mod - - bm25_root = tmp_path / "bm25_root" - monkeypatch.setattr(bm25_mod, "BM25_INDEX_ROOT", bm25_root) - nb = nb_base / "demo-nb" - _write_lancedb(nb / "lancedb", version=369) - _write_lancedb(nb / "lancedb-staging", version=645) - - nc.perform_cutover("demo-nb", base=nb_base) - - # Swap happened, but no BM25 dir was created anywhere. - assert (nb / "lancedb" / "_sentinel.txt").read_text() == "v645" - assert not bm25_root.exists() or not any(bm25_root.iterdir()) - - def test_backup_retention_prunes_to_two(nb_base) -> None: """AC6: after repeated cutovers, at most N=2 lancedb-prev-* remain.""" nb = nb_base / "demo-nb" @@ -175,15 +154,17 @@ def test_derived_indexes_travel_with_their_dataset(nb_base) -> None: nb = nb_base / "demo-nb" for name, version in (("lancedb", 369), ("lancedb-staging", 645)): _write_lancedb(nb / name, version=version) - (nb / name / "derived").mkdir() + (nb / name / "derived" / "bm25" / f"v{version}").mkdir(parents=True) (nb / name / "derived" / "theorem_names.db").write_text(f"v{version}") - index = nb / "lancedb" / "derived" / "theorem_names.db" + derived = nb / "lancedb" / "derived" nc.perform_cutover("demo-nb", base=nb_base) - assert index.read_text() == "v645" + assert (derived / "theorem_names.db").read_text() == "v645" + assert [p.name for p in (derived / "bm25").iterdir()] == ["v645"] nc.perform_rollback("demo-nb", base=nb_base) - assert index.read_text() == "v369" + assert (derived / "theorem_names.db").read_text() == "v369" + assert [p.name for p in (derived / "bm25").iterdir()] == ["v369"] def test_rollback_without_backup_refused(nb_base) -> None: diff --git a/tests/test_server_startup.py b/tests/test_server_startup.py index a06d9d6b..284cdb59 100644 --- a/tests/test_server_startup.py +++ b/tests/test_server_startup.py @@ -665,27 +665,21 @@ async def _roundtrip(): # notebook-bm25-isolation-m1 — AC-2 + config field tests # ------------------------------------------------------------------ - def test_shared_config_has_none_bm25_index_root(self, monkeypatch): - """AC-2 / FM-4: when ARXMCP_NOTEBOOK is unset, bm25_index_root - stays None so the global BM25_INDEX_ROOT is resolved at call - time (preserving the conftest monkeypatch for ~40 tests).""" + def test_shared_bm25_index_root_lives_in_the_shared_dataset(self, monkeypatch): monkeypatch.delenv("ARXMCP_NOTEBOOK", raising=False) + monkeypatch.delenv("ARXMCP_BM25_INDEX_ROOT", raising=False) cfg = Config() - assert cfg.bm25_index_root is None, ( - "Non-notebook config must have bm25_index_root=None " - f"(got {cfg.bm25_index_root!r})" - ) + assert cfg.bm25_index_root == Path("var/arxmcp/index/lancedb/derived/bm25") def test_notebook_derives_per_notebook_bm25_index_root( self, notebooks_base, monkeypatch ): - """AC-1 / notebook-bm25-isolation-m1: fork-C config derives - bm25_index_root = var/arxmcp/notebooks//index/bm25, - mirroring the cache_db_path isolation pattern.""" + """A notebook-routed config reads the BM25 index inside its own + dataset.""" monkeypatch.delenv("ARXMCP_BM25_INDEX_ROOT", raising=False) _seed_notebook_marker(notebooks_base / "demo-nb" / "lancedb") cfg = Config(notebook="demo-nb") - expected = (notebooks_base / "demo-nb" / "index" / "bm25").resolve() + expected = (notebooks_base / "demo-nb" / "lancedb" / "derived" / "bm25").resolve() assert cfg.bm25_index_root == expected, ( f"Fork-C bm25_index_root should be {expected}; " f"got {cfg.bm25_index_root!r}" @@ -737,33 +731,17 @@ def test_resources_startup_boots_notebook_corpus( assert resources.config.lancedb_path == nb_lancedb.resolve() assert resources.chunks_table.count_rows() == 2 - # notebook-bm25-isolation-m1 F2: pin that fork-C STARTUP built the - # BM25 index under the PER-NOTEBOOK root (config → resources → - # BM25Phase.startup auto-build), NOT the global root. Without this - # assertion a future refactor that drops the bm25_index_root kwarg - # would auto-build into the global root and every other test would - # still pass (/readyz would still be 200) — the e3-class - # "verified by reading, not pinned" gap. + # Startup auto-builds the missing index inside the served dataset. import ingest.bm25_indexer as bm25_mod # noqa: PLC0415 nb_bm25_pkl = ( - notebooks_base - / "demo-nb" - / "index" - / "bm25" - / f"v{nb_version}" - / bm25_mod.BM25_INDEX_NAME + notebooks_base / "demo-nb" / "lancedb" / "derived" / "bm25" + / f"v{nb_version}" / bm25_mod.BM25_INDEX_NAME ) assert nb_bm25_pkl.is_file(), ( - f"fork-C startup must build BM25 under the notebook root; " + f"startup must build BM25 inside the notebook dataset; " f"missing {nb_bm25_pkl}" ) - # The global (conftest-patched) BM25 root must NOT carry this - # notebook's version dir — proves the per-notebook isolation. - assert not (bm25_mod.BM25_INDEX_ROOT / f"v{nb_version}").exists(), ( - "fork-C must NOT build into the global BM25 root " - f"({bm25_mod.BM25_INDEX_ROOT})" - ) class TestNotebookLancedbPathHelper: diff --git a/tests/tools/test_notebook_scripts.py b/tests/tools/test_notebook_scripts.py index c17209ee..2c46a5a8 100644 --- a/tests/tools/test_notebook_scripts.py +++ b/tests/tools/test_notebook_scripts.py @@ -664,9 +664,8 @@ def test_ingest_builds_bm25_under_per_notebook_root( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - """notebook-bm25-isolation-m1 regression: build_bm25_index is called - with index_root=/index/bm25, NOT the global BM25_INDEX_ROOT. - Two notebooks at the same version N get separate roots with no collision.""" + """build_bm25_index writes into derived/bm25 inside the notebook's + dataset, where the server reads it and cutover moves it.""" nb = _seed_notebook(notebooks_base, "myslug", ["2303.07061"]) class _FakeSummary: @@ -709,8 +708,8 @@ def _capture_build(*args, **kwargs): assert "myslug" in index_root_parts, ( f"index_root should be under the notebook dir; got {index_root}" ) - assert index_root_parts[-2:] == ("index", "bm25"), ( - f"index_root should be /index/bm25; got {index_root}" + assert Path(index_root) == nb / "lancedb" / "derived" / "bm25", ( + f"index_root should be /lancedb/derived/bm25; got {index_root}" ) # Confirm no .notebook_slug sentinel is written (FM-7: sentinel removed) assert not (index_root / "v5" / ".notebook_slug").exists() diff --git a/tools/notebook_cutover.py b/tools/notebook_cutover.py index fd6e7f7f..d09ef3ba 100644 --- a/tools/notebook_cutover.py +++ b/tools/notebook_cutover.py @@ -32,20 +32,9 @@ filesystem). The most recent ``N=2`` ``lancedb-prev-*`` backups are retained; older ones are pruned (prune failure is non-fatal — the swap already committed). -**BM25 is NOT built here (F1 rectification).** An earlier draft built the BM25 -index for the staging version pre-swap, but the BM25 index root -(``var/arxmcp/index/bm25/v/``) is GLOBAL and keyed only on the per-dataset -MVCC ``corpus_version`` — which is NOT globally unique across notebooks + the -shared corpus (collision confirmed live: shared-corpus ``v49`` and -shimura-varieties active are both v49). Building into that namespace from the -cutover participates in a fork-C-startup collision. The fork-A -(``filters.notebook``) retrieval path is dense-only (no BM25); the fork-C -(``ARXMCP_NOTEBOOK``) startup path AUTO-BUILDS the BM25 index for the active -version if missing (E04_S04 H1), so the cutover does not need to. The proper -fix — a per-notebook BM25 root coordinated with fork-C startup -(``server/resources.py``) — is the BM25 analog of m1's ``cache_db_path`` -isolation and is tracked as a separate follow-up (see notebook-cutover-m1 -critique F1). +The dataset carries its derived indexes (BM25, theorem names) under +``derived/``, so the two renames move them with the chunks and a rollback +restores the ones built for the restored version. """ from __future__ import annotations @@ -175,8 +164,7 @@ def perform_cutover( Then perform the two-rename swap with an EXDEV guard + rollback-on-step-2-failure, and prune backups to ``N=2`` (prune failure is - non-fatal — the swap already committed). BM25 is intentionally NOT built - here — see the module docstring F1 note. + non-fatal — the swap already committed). """ validate_slug(slug) # Threat-1 boundary, before any path construction. nb = notebook_dir(slug, base=base) diff --git a/tools/notebook_ingest.py b/tools/notebook_ingest.py index d57ed253..736f4329 100644 --- a/tools/notebook_ingest.py +++ b/tools/notebook_ingest.py @@ -6,25 +6,15 @@ ``lancedb_staging_path=var/arxmcp/notebooks//lancedb``. 2. Reads the resulting ``corpus-version.json`` to find the per-notebook corpus version. -3. Calls :func:`ingest.bm25_indexer.build_bm25_index` with that version, - writing the artifact under ``var/arxmcp/notebooks//index/bm25/`` - (the per-notebook BM25 root). +3. Builds the BM25, definitions and theorem-name indexes for that + version inside the dataset, under ``lancedb/derived/``, so a cutover + or rollback moves them with the chunks. The brief's ``ARXMCP_LANCEDB_PATH=...`` wording was wrong — that's the SERVER's env var, not a bulk_ingest one. ``bulk_ingest`` uses the ``--lancedb-staging-path`` CLI argument / ``lancedb_staging_path`` Python keyword. See synthesis "Disagreement 4" resolution. -BM25 output path: notebook-bm25-isolation-m1 isolates the BM25 index -under ``var/arxmcp/notebooks//index/bm25/v/``, NOT the global -``var/arxmcp/index/bm25/``. The per-notebook root mirrors the existing -``lancedb_path`` / ``cache_db_path`` fork-C isolation in -:class:`server.config.Config`. The global BM25 root is unchanged for -the shared (non-notebook) server. A fork-C server's first startup after -this milestone rebuilds BM25 from scratch under the new per-notebook -root if the artifacts were only built pre-fix (FM-1: first-boot auto-build -via :meth:`server.retrieval.bm25.BM25Phase._sync_startup` is the safety net). - Usage: uv run python tools/notebook_ingest.py @@ -41,7 +31,7 @@ import logging import sys -from ingest.bm25_indexer import build_bm25_index +from ingest.bm25_indexer import build_bm25_index, resolve_bm25_root from ingest.bulk_ingest import run_bulk_ingest from ingest.index_definitions import index_definitions_for_corpus from ingest.index_theorem_names import build_theorem_names_index @@ -160,23 +150,15 @@ def run(slug: str) -> int: return 1 corpus_version = _read_corpus_version(lancedb_path) - # Per-notebook BM25 root: isolated under the notebook directory so - # two notebooks at the same corpus_version N do NOT collide - # (notebook-bm25-isolation-m1). The global BM25 root - # (var/arxmcp/index/bm25/) is untouched — the shared server still - # finds its own artifacts there. The sentinel workaround - # (.notebook_slug file) is removed because the directory namespaces - # are now 1:1 by construction (FM-7 resolution). - bm25_root = nb_dir / "index" / "bm25" + # The BM25 root sits inside the dataset, where the server reads it. + bm25_root = resolve_bm25_root(lancedb_path) bm25_v_dir = bm25_root / f"v{corpus_version}" logger.info( "building BM25 for slug=%s lancedb=%s corpus_version=%d root=%s", slug, lancedb_path, corpus_version, bm25_root, ) build_bm25_index( - str(lancedb_path), - corpus_version=corpus_version, - index_root=bm25_root, + str(lancedb_path), corpus_version=corpus_version, index_root=bm25_root, ) print( f"BM25 built for corpus_version={corpus_version} "