From 1c9f8af5c2ecab74600f1171826fea2dc52397e6 Mon Sep 17 00:00:00 2001 From: ContinuityOS Assistant Date: Wed, 14 Oct 2026 14:49:47 +0700 Subject: [PATCH 1/2] fix(mcp): require explicit opt-in for optional embedder loading --- continuityos/mcp_server.py | 22 ++- docs/MCP_OFFLINE_EMBEDDER_R1.md | 100 ++++++++++ tests/test_product_mcp_offline.py | 296 ++++++++++++++++++++++++++++++ 3 files changed, 413 insertions(+), 5 deletions(-) create mode 100644 docs/MCP_OFFLINE_EMBEDDER_R1.md create mode 100644 tests/test_product_mcp_offline.py diff --git a/continuityos/mcp_server.py b/continuityos/mcp_server.py index bf255760..5b2538a2 100644 --- a/continuityos/mcp_server.py +++ b/continuityos/mcp_server.py @@ -112,6 +112,13 @@ def __init__( r15_controller_profile_sha256: str = "", r15_custody_path: str = "", ): + # Match core cos: optional model construction requires explicit opt-in. + # The existing CURRENT wrapper denies before this constructor is entered. + embedder_mode = os.environ.get("CONTINUITYOS_EMBEDDER", "").strip().lower() + if embedder_mode not in { + "", "hash", "hashing", "offline", "local", "fast", "fastembed", + }: + raise ValueError("UNSUPPORTED_EMBEDDER_MODE") resolved = resolve_memory_db(db) db = resolved["path"] self.db_path = db @@ -120,11 +127,16 @@ def __init__( and db != ":memory:" and not os.path.isfile(db) ) - try: - from .embedders import FastEmbedEmbedder - self.m = Memory(db, embedder=FastEmbedEmbedder()) - except Exception: - self.m = Memory(db) # fallback to HashingEmbedder + embedder = None + if embedder_mode in {"fast", "fastembed"}: + try: + from .embedders import FastEmbedEmbedder + embedder = FastEmbedEmbedder() + except Exception: + # Preserve the legacy opt-in loader-failure fallback only. + pass + # Memory errors must propagate, not reopen the same DB with another model. + self.m = Memory(db, embedder=embedder) self.c = Continuity(memory=self.m) self.c._context_source = db_source or resolved["source"] self._governance_context_error = ( diff --git a/docs/MCP_OFFLINE_EMBEDDER_R1.md b/docs/MCP_OFFLINE_EMBEDDER_R1.md new file mode 100644 index 00000000..6acca247 --- /dev/null +++ b/docs/MCP_OFFLINE_EMBEDDER_R1.md @@ -0,0 +1,100 @@ +# MCP offline-first embedder selection — R1 + +Specification status: RC. Scope: the existing stdio MCP server, not a replacement +runtime, permission issuer, model router, schema migration, or OS sandbox. +Predecessor: `34d02e44104146beba7b174a88d58270606891fd` (A4/A3, PR #214). +Implementation and test qualification belong to the exact local commit and its +separate evidence manifest; this document is not production admission. + +## Why this change + +The predecessor constructed `FastEmbedEmbedder()` on every `Server` startup and +fell back to `Memory(db)` after any exception in that block. This included calls +with `CONTINUITYOS_EMBEDDER=hash` or `offline`. Core `cos` already requires explicit +opt-in for optional model construction. A synthetic constructor sentinel and +actual stdio child processes reproduce the difference without a real model. + +## Selection contract + +The server samples `CONTINUITYOS_EMBEDDER` once on construction, strips surrounding +whitespace, and lowercases it. It does not cache the setting process-wide or +change the embedder of an already-created server. + +| Environment value | Behavior | +| --- | --- | +| Unset, empty, whitespace only | `Memory` uses its local `HashingEmbedder` default | +| `hash`, `hashing`, `offline`, `local` (case-insensitive) | Same local default; no optional constructor | +| `fast`, `fastembed` (case-insensitive) | Explicit opt-in: attempt the existing optional constructor | +| Any other value | Raise `ValueError("UNSUPPORTED_EMBEDDER_MODE")` before DB resolution/opening | + +Offline modes do not import `continuityos.embedders` on this startup branch or +instantiate FastEmbed. Other application code may independently import modules; +this is not a blanket import prohibition. + +For explicit fast opt-in only, constructor/import failure retains the prior +fallback to the local default. This does **not** prove that an existing vector +index is compatible with a different embedder, and is not a migration mechanism. +An operator relying on FastEmbed must explicitly select it and separately verify +backend availability and the matching existing embedding space. + +`Memory` is constructed exactly once after selection. An exception from Memory +initialization is propagated; it must not trigger reopening the same database +with another embedder. Constructor-failure fallback is not permission to bypass +Memory, store, path, or effect guards. + +## CURRENT and authority boundaries + +The existing direct-surface wrapper remains outside `Server.__init__`. A declared +CURRENT binding must still pass the existing boundary before startup. Invalid or +incomplete declarations cannot become a legacy fallback. Module execution via +`python -m continuityos.mcp_server` remains subject to its existing guard. + +No CURRENT checker, effect ceiling, R14 witness, R15 runtime binding, policy +snapshot, database resolver, broker registry, transaction, execution path, or +JSON-RPC tool schema is changed. Unknown embedder mode is validated before DB +resolution; the previously loaded policy is never swapped to make a test pass. + +`fast` is a choice of optional loader, not authorization for a forbidden current +session, real personal-memory disclosure, trading, network access, or production. +The broader mandatory-path/OS-isolation gate remains unqualified. + +## Verification + +`tests/test_product_mcp_offline.py` covers: + +- Default/offline aliases with a constructor sentinel; fast aliases with a + recording fake backend; constructor-failure fallback. +- Invalid mode before resolver, Memory, loader, or database effect; existing + synthetic database preservation on rejection. +- One Memory construction even when it fails; separate instances independently + sample mode; exact existing DB and one policy snapshot reach the broker. +- Declared-current rejection for direct construction and `main`, including + unknown mode and fast opt-in. +- Actual stdio process startup, remembering a synthetic marker, and rejection + before protocol/DB initialization; a local fake `fastembed` module makes both + optional import and construction observable without downloading a model. + +The same new test bodies are run against the unchanged predecessor first and +then against the patch. Existing broker/MCP tests and current-boundary/Person +regressions remain unmodified. Optional-available qualification uses a synthetic +sentinel package, not a live ONNX model. Optional-absent qualification uses a +minimal disposable Python environment with no shared site-packages or pip fetch. +A test-local Python audit denies socket creation, DNS and connections, allowing +only local hostname lookup for JUnit metadata. It is not OS-level isolation. + +## Compatibility and limits + +The default choice changes on hosts with FastEmbed installed: installing an +optional dependency alone no longer activates it in MCP. Existing FastEmbed users +must set `CONTINUITYOS_EMBEDDER=fast` or `fastembed`. No records are re-embedded, +no vector metadata is silently rewritten, and no active installation is switched +by publishing or testing this source change. + +The plain Python API raises on unknown mode. The unbound stdio entry point exits +nonzero before serving JSON-RPC; it does not emit a successful initialization or +silently substitute hash/fast. Existing bound-session errors retain precedence. + +No live-model/download, real-data, cross-machine, packaged-release, or independent +Opus qualification is claimed by the local sentinel tests. No push, PR change, +merge, deployment, scheduler, service, TPM, OS account, or ACL operation is part +of this local work order. `C0 HOLD`; `NOT_PRODUCTION_ADMITTED`. diff --git a/tests/test_product_mcp_offline.py b/tests/test_product_mcp_offline.py new file mode 100644 index 00000000..cfe1b712 --- /dev/null +++ b/tests/test_product_mcp_offline.py @@ -0,0 +1,296 @@ +"""Offline-first MCP selection; optional backends are synthetic sentinels only.""" +from __future__ import annotations + +from contextlib import contextmanager +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from continuityos import current_effect_boundary, embedders, mcp_server +from continuityos.embed import HashingEmbedder +from continuityos.memory import Memory + + +PACKAGE_ROOT = Path(mcp_server.__file__).resolve().parents[1] +OFFLINE_MODES = [None, "", " ", "hash", "HASH", "hashing", " HASHING ", + "offline", " Offline ", "local", "LOCAL"] +FAST_MODES = ["fast", " FAST ", "fastembed", "FastEmbed"] + + +@pytest.fixture(autouse=True) +def isolated_environment(monkeypatch, tmp_path): + for key in tuple(os.environ): + if key.startswith("CONTINUITYOS_CURRENT_"): + monkeypatch.delenv(key) + monkeypatch.delenv("CONTINUITYOS_EMBEDDER", raising=False) + home = tmp_path / "home" + home.mkdir() + for key in ("HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA"): + monkeypatch.setenv(key, str(home)) + monkeypatch.setenv("CONTINUITYOS_SILENCE_EMBED_WARN", "1") + + def no_real_backend(*args, **kwargs): + raise AssertionError("A test must never construct the real optional backend") + + monkeypatch.setattr(embedders, "FastEmbedEmbedder", no_real_backend) + + +def set_mode(monkeypatch, mode): + if mode is None: + monkeypatch.delenv("CONTINUITYOS_EMBEDDER", raising=False) + else: + monkeypatch.setenv("CONTINUITYOS_EMBEDDER", mode) + + +@contextmanager +def opened_server(db, policy=""): + server = mcp_server.Server(str(db), str(policy)) + try: + yield server + finally: + server.m.store.con.close() + + +@pytest.mark.parametrize("mode", OFFLINE_MODES) +def test_offline_modes_never_construct_optional_backend(monkeypatch, tmp_path, mode): + set_mode(monkeypatch, mode) + calls = [] + + def forbidden(*args, **kwargs): + calls.append("constructed") + raise AssertionError("No optional model in an offline mode") + + monkeypatch.setattr(embedders, "FastEmbedEmbedder", forbidden) + with opened_server(tmp_path / "memory.db") as server: + assert isinstance(server.m.embed, HashingEmbedder) + assert calls == [] + + +@pytest.mark.parametrize("mode", FAST_MODES) +def test_fast_aliases_require_and_honor_explicit_opt_in(monkeypatch, tmp_path, mode): + set_mode(monkeypatch, mode) + instances = [] + + class FakeFast: + def __init__(self): + instances.append(self) + + def __call__(self, text): + return HashingEmbedder()(text) + + monkeypatch.setattr(embedders, "FastEmbedEmbedder", FakeFast) + with opened_server(tmp_path / "memory.db") as server: + assert len(instances) == 1 + assert server.m.embed is instances[0] + assert server.db_path == os.path.normcase(os.path.realpath(tmp_path / "memory.db")) + + +@pytest.mark.parametrize("failure", [ImportError, RuntimeError]) +def test_opted_in_constructor_failure_keeps_legacy_hash_fallback(monkeypatch, tmp_path, failure): + set_mode(monkeypatch, "fast") + calls = [] + + def unavailable(*args, **kwargs): + calls.append("constructor") + raise failure("synthetic unavailable optional loader") + + monkeypatch.setattr(embedders, "FastEmbedEmbedder", unavailable) + with opened_server(tmp_path / "memory.db") as server: + assert calls == ["constructor"] + assert isinstance(server.m.embed, HashingEmbedder) + + +@pytest.mark.parametrize("mode", ["auto", "unknown", "0", "hash+fast", "fast\nhash", "fast embed"]) +def test_unknown_mode_fails_before_resolution_memory_or_optional_loader(monkeypatch, tmp_path, mode): + set_mode(monkeypatch, mode) + touched = [] + + def forbidden(*args, **kwargs): + touched.append("resolution-or-loader-or-memory") + raise AssertionError("Unknown mode reached an effectful dependency") + + monkeypatch.setattr(mcp_server, "resolve_memory_db", forbidden) + monkeypatch.setattr(mcp_server, "Memory", forbidden) + monkeypatch.setattr(embedders, "FastEmbedEmbedder", forbidden) + with pytest.raises(ValueError, match="UNSUPPORTED_EMBEDDER_MODE"): + mcp_server.Server(str(tmp_path / "must-not-exist.db")) + assert touched == [] + assert not (tmp_path / "must-not-exist.db").exists() + + +def test_unknown_mode_does_not_touch_an_existing_database(monkeypatch, tmp_path): + db = tmp_path / "existing.db" + memory = Memory(str(db)) + memory.remember("synthetic sentinel", namespace="facts") + memory.store.con.close() + before = hashlib.sha256(db.read_bytes()).hexdigest() + set_mode(monkeypatch, "unsupported") + try: + with opened_server(db): + pytest.fail("Unknown mode was accepted") + except ValueError as exc: + assert "UNSUPPORTED_EMBEDDER_MODE" in str(exc) + assert hashlib.sha256(db.read_bytes()).hexdigest() == before + assert not Path(str(db) + "-wal").exists() + + +@pytest.mark.parametrize("mode", ["hash", "fast"]) +def test_memory_initialization_failure_is_not_retried_with_another_embedder(monkeypatch, tmp_path, mode): + set_mode(monkeypatch, mode) + backend = object() + memory_calls = [] + monkeypatch.setattr(embedders, "FastEmbedEmbedder", lambda: backend) + + def failed_memory(*args, **kwargs): + memory_calls.append((args, kwargs)) + raise OSError("SYNTHETIC_MEMORY_OPEN_FAILED") + + monkeypatch.setattr(mcp_server, "Memory", failed_memory) + with pytest.raises(OSError, match="SYNTHETIC_MEMORY_OPEN_FAILED"): + mcp_server.Server(str(tmp_path / "unopened.db")) + assert len(memory_calls) == 1 + assert not (tmp_path / "unopened.db").exists() + + +def test_mode_is_sampled_once_per_server_not_cached_globally(monkeypatch, tmp_path): + fake = HashingEmbedder() + calls = [] + + def constructor(): + calls.append("fast") + return fake + + monkeypatch.setattr(embedders, "FastEmbedEmbedder", constructor) + set_mode(monkeypatch, "hash") + with opened_server(tmp_path / "one.db") as first: + first_embedder = first.m.embed + set_mode(monkeypatch, "fast") + with opened_server(tmp_path / "two.db") as second: + assert second.m.embed is fake + assert first.m.embed is first_embedder and first_embedder is not fake + set_mode(monkeypatch, "local") + with opened_server(tmp_path / "three.db") as third: + assert third.m.embed is not fake + assert calls == ["fast"] + + +def test_existing_policy_snapshot_and_db_binding_are_preserved(monkeypatch, tmp_path): + set_mode(monkeypatch, "hash") + db = tmp_path / "existing.db" + memory = Memory(str(db)) + memory.store.con.close() + policy = tmp_path / "policy.json" + policy.write_text(json.dumps({"default_decision": "DENY"}), encoding="utf-8") + with opened_server(db, policy) as server: + broker = server._gate_broker() + assert broker.db == server.db_path == os.path.normcase(os.path.realpath(db)) + assert broker._policy_snapshot is server.policy + assert server.policy["default_decision"] == "DENY" + policy.write_text(json.dumps({"default_decision": "ALLOW"}), encoding="utf-8") + assert server.policy["default_decision"] == "DENY" + + +@pytest.mark.parametrize("mode", ["hash", "fast", "unknown"]) +@pytest.mark.parametrize("surface", ["Server", "main"]) +def test_declared_current_session_denies_before_embedder_or_db(monkeypatch, tmp_path, mode, surface): + set_mode(monkeypatch, mode) + monkeypatch.setenv("CONTINUITYOS_CURRENT_SESSION_REQUIRED", "1") + touched = [] + + def forbidden(*args, **kwargs): + touched.append("effect") + raise AssertionError("CURRENT must deny before resolution or loader") + + monkeypatch.setattr(mcp_server, "resolve_memory_db", forbidden) + monkeypatch.setattr(mcp_server, "Memory", forbidden) + monkeypatch.setattr(embedders, "FastEmbedEmbedder", forbidden) + with pytest.raises(current_effect_boundary.CurrentEffectBoundaryError): + if surface == "Server": + mcp_server.Server(str(tmp_path / "unopened.db")) + else: + mcp_server.main() + assert touched == [] + assert not (tmp_path / "unopened.db").exists() + + +def run_stdio_process(tmp_path, mode, *, required=False): + stub = tmp_path / "optional-stub" + stub.mkdir() + imported = tmp_path / "optional-imported.txt" + constructed = tmp_path / "optional-constructed.txt" + (stub / "fastembed.py").write_text( + "import os\nfrom pathlib import Path\n" + "Path(os.environ['COS_TEST_OPTIONAL_IMPORTED']).write_text('imported')\n" + "class TextEmbedding:\n" + " def __init__(self, **kwargs):\n" + " Path(os.environ['COS_TEST_OPTIONAL_CONSTRUCTED']).write_text('constructed')\n" + " def embed(self, texts):\n" + " for text in texts: yield [1.0] + [0.0] * 383\n", + encoding="utf-8", + ) + env = {k: v for k, v in os.environ.items() if k.upper() in { + "SYSTEMROOT", "WINDIR", "COMSPEC", "PATH", "PATHEXT" + }} + home = tmp_path / "child-home" + home.mkdir() + env.update({ + "HOME": str(home), "USERPROFILE": str(home), "APPDATA": str(home), + "LOCALAPPDATA": str(home), "TEMP": str(tmp_path), "TMP": str(tmp_path), + "PYTHONPATH": str(stub) + os.pathsep + str(PACKAGE_ROOT), + "PYTHONDONTWRITEBYTECODE": "1", "CONTINUITYOS_SILENCE_EMBED_WARN": "1", + "COS_TEST_OPTIONAL_IMPORTED": str(imported), + "COS_TEST_OPTIONAL_CONSTRUCTED": str(constructed), + }) + if mode is not None: + env["CONTINUITYOS_EMBEDDER"] = mode + if required: + env["CONTINUITYOS_CURRENT_SESSION_REQUIRED"] = "1" + db = tmp_path / "stdio.db" + requests = [ + {"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}, + {"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": { + "name": "remember", "arguments": {"text": "synthetic MCP offline marker"}}}, + ] + process = subprocess.run( + [sys.executable, "-B", "-m", "continuityos.mcp_server", "--db", str(db)], + input="".join(json.dumps(r) + "\n" for r in requests), + cwd=str(tmp_path), env=env, capture_output=True, + text=True, encoding="utf-8", timeout=25, + ) + return process, db, imported, constructed + + +@pytest.mark.parametrize("mode", [None, "hash", "local", "fast", "fastembed"]) +def test_real_stdio_entrypoint_obeys_mode_without_real_model(tmp_path, mode): + process, db, imported, constructed = run_stdio_process(tmp_path, mode) + assert process.returncode == 0, process.stderr + responses = [json.loads(line) for line in process.stdout.splitlines() if line.strip()] + assert [r["id"] for r in responses] == [1, 2] + assert not responses[1]["result"].get("isError") + assert "stored #" in responses[1]["result"]["content"][0]["text"] + assert db.exists() + opted_in = mode in {"fast", "fastembed"} + assert imported.exists() is opted_in + assert constructed.exists() is opted_in + + +def test_real_stdio_invalid_mode_exits_before_protocol_and_database(tmp_path): + process, db, imported, constructed = run_stdio_process(tmp_path, "unknown") + assert process.returncode != 0 + assert "UNSUPPORTED_EMBEDDER_MODE" in process.stderr + assert not process.stdout.strip() + assert not db.exists() and not imported.exists() and not constructed.exists() + + +@pytest.mark.parametrize("mode", ["hash", "fast", "unknown"]) +def test_real_stdio_current_boundary_precedes_mode_selection(tmp_path, mode): + process, db, imported, constructed = run_stdio_process(tmp_path, mode, required=True) + assert process.returncode != 0 + assert "CurrentEffectBoundaryError" in process.stderr + assert not process.stdout.strip() + assert not db.exists() and not imported.exists() and not constructed.exists() From 0bd9d58c898a9756d9b6986b8971b9b6a89fcd01 Mon Sep 17 00:00:00 2001 From: bitmaster162 Date: Wed, 14 Oct 2026 16:44:44 +0700 Subject: [PATCH 2/2] fix(mcp): reject incompatible vectors and explicit backend failure --- continuityos/mcp_server.py | 44 +++- docs/MCP_EMBEDDER_COMPATIBILITY_R2.md | 95 ++++++++ tests/test_product_mcp_offline.py | 308 +++++++++++++++++++++++++- 3 files changed, 439 insertions(+), 8 deletions(-) create mode 100644 docs/MCP_EMBEDDER_COMPATIBILITY_R2.md diff --git a/continuityos/mcp_server.py b/continuityos/mcp_server.py index 5b2538a2..55728992 100644 --- a/continuityos/mcp_server.py +++ b/continuityos/mcp_server.py @@ -132,11 +132,49 @@ def __init__( try: from .embedders import FastEmbedEmbedder embedder = FastEmbedEmbedder() - except Exception: - # Preserve the legacy opt-in loader-failure fallback only. - pass + except Exception as exc: + raise RuntimeError( + "FAST_EMBEDDER_UNAVAILABLE: explicit fast mode cannot fall back to hash" + ) from exc # Memory errors must propagate, not reopen the same DB with another model. self.m = Memory(db, embedder=embedder) + try: + # Synthetic probe only: never send a stored record to the embedder here. + try: + probe = self.m.embed("continuityos:mcp-embedding-dimension-probe:v1") + except Exception as exc: + raise ValueError("EMBEDDER_PROBE_FAILED") from exc + if not isinstance(probe, (list, tuple)) or not probe: + raise ValueError("EMBEDDER_PROBE_INVALID") + selected_dimension = len(probe) + # All non-NULL vectors, across namespaces; no text or BLOB payload reads. + cursor = self.m.store.con.execute( + "SELECT MIN(length(vec)), MAX(length(vec)), " + "COALESCE(SUM(CASE WHEN typeof(vec) <> 'blob' " + "OR length(vec) = 0 OR length(vec) % 4 <> 0 " + "THEN 1 ELSE 0 END), 0) FROM items WHERE vec IS NOT NULL" + ) + try: + minimum_bytes, maximum_bytes, malformed = cursor.fetchone() + finally: + cursor.close() + if malformed: + raise ValueError("EMBEDDER_VECTOR_DATA_INVALID: non-empty float32 BLOB required") + if minimum_bytes is not None and ( + minimum_bytes != maximum_bytes or minimum_bytes != selected_dimension * 4 + ): + stored = str(minimum_bytes // 4) + if minimum_bytes != maximum_bytes: + stored += ".." + str(maximum_bytes // 4) + " (mixed)" + raise ValueError( + f"EMBEDDER_DIMENSION_MISMATCH store={stored} selected={selected_dimension}; " + "set CONTINUITYOS_EMBEDDER to the original compatible backend; " + "restore or re-embed separately, never mix vector spaces" + ) + except BaseException: + # A failed startup must not leave its opened Memory connection alive. + self.m.store.con.close() + raise self.c = Continuity(memory=self.m) self.c._context_source = db_source or resolved["source"] self._governance_context_error = ( diff --git a/docs/MCP_EMBEDDER_COMPATIBILITY_R2.md b/docs/MCP_EMBEDDER_COMPATIBILITY_R2.md new file mode 100644 index 00000000..e769f9a7 --- /dev/null +++ b/docs/MCP_EMBEDDER_COMPATIBILITY_R2.md @@ -0,0 +1,95 @@ +# MCP embedding compatibility R2 + +Status: local corrective candidate; no activation or production admission. +Base: `1c9f8af5c2ecab74600f1171826fea2dc52397e6`, the reviewed PR #215 head. +This document supersedes only the R1 loader-failure fallback and startup +compatibility behavior. `MCP_OFFLINE_EMBEDDER_R1.md` remains the historical R1 +contract; it has not been silently rewritten. + +## Findings and scope + +The owner-delivered Claude/Opus review identified two activation blockers. +Synthetic execution reproduced both: the R1 MCP default against a 384-D store +admitted a new 256-D row; a subsequent 384-D backend still failed when recall +included both dimensions. Explicit fast constructor failure could silently select +Hashing, even in an empty database. One-row sampling can miss existing mixtures. + +This correction covers startup on the existing MCP path. It is not a Store write +guard, an embedder identity registry, a new runtime, or a database migration. + +## Selection and failure contract + +Mode normalization and the R1 aliases are unchanged. Default, empty, whitespace, +hash/hashing/offline/local do not import or construct FastEmbed. Only explicit +fast/fastembed selects the optional loader. Unknown modes raise +`UNSUPPORTED_EMBEDDER_MODE` before DB resolution. CURRENT containment retains +precedence; its implementation is unchanged. + +**Intentional R1 contract change:** an explicitly selected FastEmbed import or +constructor failure raises `FAST_EMBEDDER_UNAVAILABLE`. No Hashing fallback, +warning-only degradation, or second Memory-open attempt is performed. The chained +exception retains the original failure. Path resolution may already have happened, +but Memory is not constructed on this failure path. + +## Dimension barrier + +After the single Memory construction, and before Continuity/Twin/control-plane +initialization or any JSON-RPC response, the server: + +1. Calls the selected embedder once with the fixed synthetic probe + `continuityos:mcp-embedding-dimension-probe:v1`. No stored record is supplied. + Exceptions produce `EMBEDDER_PROBE_FAILED`; a non-list/tuple or empty result + produces `EMBEDDER_PROBE_INVALID`. +2. Issues one SELECT for minimum/maximum vector byte length and a malformed count + across all non-NULL `items.vec`. No text, namespace, or BLOB payload is returned + to Python. Equal valid float32 widths imply a uniform stored dimension. + Empty and NULL-only stores impose no stored-dimension constraint. +3. Rejects non-BLOB, empty, or non-float32-aligned payloads with + `EMBEDDER_VECTOR_DATA_INVALID`. Mixed or incompatible widths produce + `EMBEDDER_DIMENSION_MISMATCH store= selected=` + and an operator hint. No records are deleted or repaired. +4. Closes the opened Memory connection and propagates any failure in this new + validation block. Query failures remain failures; they do not mean empty store. + +The aggregate reads all vector metadata and returns one row. Its scan cost grows +with store size. A mixed range in the error is min/max, not an enumeration of every +intermediate size. Vector lengths are float32 BLOB bytes divided by four. + +Memory retains its existing initialization behavior, including WAL/schema work +where applicable. This check is before serving and adding a memory item, not a +claim that constructing arbitrary legacy Memory has zero filesystem effects. +A newly created empty database can exist if the later probe fails. No new schema, +re-embedding, automatic migration or repair is introduced. + +## Compatibility and activation + +HashingEmbedder uses 256 dimensions; the current FastEmbed bge-small backend uses +384. A 384-D database started in default/hash mode now fails before the first MCP +write. Switching back to fast does not repair already mixed rows: that store also +fails startup until separately restored or rebuilt under an approved procedure. + +Vector length alone must not be used to infer backend identity. Two 256-D models +can inhabit different coordinate spaces. The barrier checks dimensions and BLOB +shape, not model identity, numeric finiteness, quality, provenance or authority. +Explicit fast mode may initialize a real optional model and then run the probe; +qualification tests use synthetic backends with network denied, not real models. + +## Limits and regression contract + +This is a startup snapshot, not a cross-process lock or continuous write invariant. +A raw Memory/Store writer can insert incompatible data after startup. A backend +whose output shape changes later is not constrained by this initial probe. +Core cos selection, Memory/Store/index, R14/R15, Person R21H and A4/A3 are unchanged. +Shared mode selection and full embedder identity are separate proposals. + +R2 explicitly replaces the two parameterized R1 loader-failure fallback cases +with fail-closed expectations. Other original test bodies are preserved; the stdio +utility gains an optional synthetic import-failure mode. New tests cover all-row +versus one-row checking, order/namespace independence, malformed BLOBs, probe/query +errors and connection closure, compatible stores, fast failure, and real stdio +refusal before initialize/remember responses. + +The prior Opus REVISE supports the findings; it is not approval of this correction. +Independent review and a new official wheel/CI remain required. No push, PR edit, +merge, deployment, runtime switch, real-user-data access, model download or trading +is part of this local work. diff --git a/tests/test_product_mcp_offline.py b/tests/test_product_mcp_offline.py index cfe1b712..6ccea16d 100644 --- a/tests/test_product_mcp_offline.py +++ b/tests/test_product_mcp_offline.py @@ -91,7 +91,8 @@ def __call__(self, text): @pytest.mark.parametrize("failure", [ImportError, RuntimeError]) -def test_opted_in_constructor_failure_keeps_legacy_hash_fallback(monkeypatch, tmp_path, failure): +def test_explicit_fast_failure_is_closed_before_memory(monkeypatch, tmp_path, failure): + # R2 deliberately replaces the R1 fallback contract following Claude review. set_mode(monkeypatch, "fast") calls = [] @@ -100,9 +101,11 @@ def unavailable(*args, **kwargs): raise failure("synthetic unavailable optional loader") monkeypatch.setattr(embedders, "FastEmbedEmbedder", unavailable) - with opened_server(tmp_path / "memory.db") as server: - assert calls == ["constructor"] - assert isinstance(server.m.embed, HashingEmbedder) + with pytest.raises(RuntimeError, match="FAST_EMBEDDER_UNAVAILABLE"): + with opened_server(tmp_path / "memory.db"): + pass + assert calls == ["constructor"] + assert not (tmp_path / "memory.db").exists() @pytest.mark.parametrize("mode", ["auto", "unknown", "0", "hash+fast", "fast\nhash", "fast embed"]) @@ -218,7 +221,7 @@ def forbidden(*args, **kwargs): assert not (tmp_path / "unopened.db").exists() -def run_stdio_process(tmp_path, mode, *, required=False): +def run_stdio_process(tmp_path, mode, *, required=False, loader_failure=False): stub = tmp_path / "optional-stub" stub.mkdir() imported = tmp_path / "optional-imported.txt" @@ -233,6 +236,10 @@ def run_stdio_process(tmp_path, mode, *, required=False): " for text in texts: yield [1.0] + [0.0] * 383\n", encoding="utf-8", ) + if loader_failure: + (stub / "fastembed.py").write_text( + "raise ImportError('SYNTHETIC_FASTEMBED_IMPORT_FAILURE')\n", encoding="utf-8", + ) env = {k: v for k, v in os.environ.items() if k.upper() in { "SYSTEMROOT", "WINDIR", "COMSPEC", "PATH", "PATHEXT" }} @@ -294,3 +301,294 @@ def test_real_stdio_current_boundary_precedes_mode_selection(tmp_path, mode): assert "CurrentEffectBoundaryError" in process.stderr assert not process.stdout.strip() assert not db.exists() and not imported.exists() and not constructed.exists() + + +# R2: guard a complete startup snapshot, not only one sampled vector row. +def seed_vector_dimensions(db, dimensions, *, separate_namespaces=False): + memory = Memory(str(db)) + try: + for index, dimension in enumerate(dimensions): + memory.store.add( + "synthetic compatibility record " + str(index), + namespace=("space-" + str(index)) if separate_namespaces else "notes", + vec=None if dimension is None else [1.0] + [0.0] * (dimension - 1), + ) + finally: + memory.store.con.close() + + +def read_vector_snapshot(db): + import sqlite3 + connection = sqlite3.connect(db.as_uri() + "?mode=ro", uri=True) + try: + return connection.execute( + "SELECT id, namespace, text, meta, vec FROM items ORDER BY id" + ).fetchall() + finally: + connection.close() + + +def track_created_memory(monkeypatch): + created = [] + + def memory_factory(*args, **kwargs): + memory = Memory(*args, **kwargs) + created.append(memory) + return memory + + monkeypatch.setattr(mcp_server, "Memory", memory_factory) + return created + + +def assert_memory_closed(created): + import sqlite3 + assert len(created) == 1 + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + created[0].store.con.execute("SELECT 1") + + +@pytest.mark.parametrize("mode", OFFLINE_MODES) +def test_existing_384_rejected_before_first_mcp_write(monkeypatch, tmp_path, mode): + db = tmp_path / "legacy-fast.db" + seed_vector_dimensions(db, [384]) + before = read_vector_snapshot(db) + before_sha = hashlib.sha256(db.read_bytes()).hexdigest() + created = track_created_memory(monkeypatch) + set_mode(monkeypatch, mode) + with pytest.raises(ValueError, match=r"EMBEDDER_DIMENSION_MISMATCH.*store=384.*selected=256"): + with opened_server(db): + pass + assert_memory_closed(created) + # The read-only snapshot observer can itself create WAL sidecars. + assert not Path(str(db) + "-wal").exists() + assert read_vector_snapshot(db) == before + assert hashlib.sha256(db.read_bytes()).hexdigest() == before_sha + + +@pytest.mark.parametrize("mode,dimension", [(None, 256), ("fast", 384)]) +def test_compatible_backend_probed_once_without_record_content(monkeypatch, tmp_path, mode, dimension): + db = tmp_path / "compatible.db" + seed_vector_dimensions(db, [dimension, dimension]) + calls = [] + queries = [] + + class RecordingBackend: + def __call__(self, text): + calls.append(text) + return [1.0] + [0.0] * (dimension - 1) + + backend = RecordingBackend() + if mode == "fast": + monkeypatch.setattr(embedders, "FastEmbedEmbedder", lambda: backend) + + def memory_factory(*args, **kwargs): + memory = Memory(*args, **kwargs) + if mode is None: + actual = memory.embed + + def recorded_hash(text): + calls.append(text) + return actual(text) + + memory.embed = recorded_hash + memory.store.con.set_trace_callback(queries.append) + return memory + + monkeypatch.setattr(mcp_server, "Memory", memory_factory) + set_mode(monkeypatch, mode) + with opened_server(db) as server: + assert len(calls) == 1 + assert calls[0] == "continuityos:mcp-embedding-dimension-probe:v1" + reads = [q for q in queries if q.lstrip().upper().startswith("SELECT")] + assert len(reads) == 1 + assert "length(vec)" in reads[0].lower() + assert "typeof(vec)" in reads[0].lower() + assert " text" not in reads[0].lower() + assert " namespace" not in reads[0].lower() + assert "LIMIT 1" not in reads[0].upper() + assert server.m.store.count() == 2 + server.call("remember", {"text": "synthetic compatible write"}) + result = json.loads(server.call("recall", {"query": "synthetic", "k": 10})) + assert result + assert server.m.store.count() == 3 + + +@pytest.mark.parametrize("mode,dimensions", [ + ("hash", [256, 384]), ("hash", [384, 256]), + ("fast", [384, 256]), ("fast", [256, 384]), +]) +def test_all_namespaces_checked_not_one_sample(monkeypatch, tmp_path, mode, dimensions): + db = tmp_path / "mixed.db" + seed_vector_dimensions(db, dimensions, separate_namespaces=True) + before = read_vector_snapshot(db) + if mode == "fast": + monkeypatch.setattr(embedders, "FastEmbedEmbedder", lambda: (lambda text: [1.0] + [0.0] * 383)) + set_mode(monkeypatch, mode) + created = track_created_memory(monkeypatch) + with pytest.raises(ValueError, match="EMBEDDER_DIMENSION_MISMATCH"): + with opened_server(db): + pass + assert_memory_closed(created) + assert read_vector_snapshot(db) == before + + +@pytest.mark.parametrize("bad_blob", [b"", b"x", b"x" * 1025, "not-a-vector", 42]) +def test_invalid_stored_vector_shape_fails_closed(monkeypatch, tmp_path, bad_blob): + import sqlite3 + db = tmp_path / "malformed.db" + seed_vector_dimensions(db, [256]) + connection = sqlite3.connect(db) + try: + connection.execute("UPDATE items SET vec=?", (bad_blob,)) + connection.commit() + finally: + connection.close() + before = read_vector_snapshot(db) + created = track_created_memory(monkeypatch) + with pytest.raises(ValueError, match="EMBEDDER_VECTOR_DATA_INVALID"): + with opened_server(db): + pass + assert_memory_closed(created) + assert read_vector_snapshot(db) == before + + +@pytest.mark.parametrize("dimensions", [[], [None], [None, 256]]) +def test_empty_or_vectorless_store_uses_selected_backend_without_fabricated_constraint(tmp_path, dimensions): + db = tmp_path / "vectorless.db" + seed_vector_dimensions(db, dimensions) + with opened_server(db) as server: + server.call("remember", {"text": "synthetic first vector"}) + assert server.m.store.count() == len(dimensions) + 1 + assert json.loads(server.call("recall", {"query": "synthetic"})) + + +@pytest.mark.parametrize("bad_probe", [None, [], "not-a-vector", b"not-a-vector", True]) +def test_invalid_probe_closes_connection_before_serving(monkeypatch, tmp_path, bad_probe): + db = tmp_path / "probe-invalid.db" + seed_vector_dimensions(db, []) + set_mode(monkeypatch, "fast") + monkeypatch.setattr(embedders, "FastEmbedEmbedder", lambda: (lambda text: bad_probe)) + created = track_created_memory(monkeypatch) + with pytest.raises(ValueError, match="EMBEDDER_PROBE_INVALID"): + with opened_server(db): + pass + assert_memory_closed(created) + assert read_vector_snapshot(db) == [] + + +def test_probe_exception_closes_connection_without_fallback(monkeypatch, tmp_path): + db = tmp_path / "probe-error.db" + seed_vector_dimensions(db, [384]) + before = read_vector_snapshot(db) + calls = [] + + def probe(text): + calls.append(text) + raise RuntimeError("SYNTHETIC_PROBE_FAILED") + + set_mode(monkeypatch, "fast") + monkeypatch.setattr(embedders, "FastEmbedEmbedder", lambda: probe) + created = track_created_memory(monkeypatch) + with pytest.raises(ValueError, match="EMBEDDER_PROBE_FAILED"): + with opened_server(db): + pass + assert len(calls) == 1 + assert_memory_closed(created) + assert read_vector_snapshot(db) == before + + +def test_vector_scan_error_closes_connection_and_never_serves(monkeypatch, tmp_path): + import sqlite3 + db = tmp_path / "scan-error.db" + seed_vector_dimensions(db, [256]) + real_connections = [] + attempts = [] + + class FailingRead: + def __init__(self, connection): + self.connection = connection + + def execute(self, sql, *args): + attempts.append(sql) + raise sqlite3.OperationalError("SYNTHETIC_DIMENSION_SCAN_FAILED") + + def close(self): + self.connection.close() + + def memory_factory(*args, **kwargs): + memory = Memory(*args, **kwargs) + real_connections.append(memory.store.con) + memory.store.con = FailingRead(memory.store.con) + return memory + + monkeypatch.setattr(mcp_server, "Memory", memory_factory) + with pytest.raises(sqlite3.OperationalError, match="SYNTHETIC_DIMENSION_SCAN_FAILED"): + with opened_server(db): + pass + assert len(real_connections) == 1 and len(attempts) == 1 + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + real_connections[0].execute("SELECT 1") + + +@pytest.mark.parametrize("mode", FAST_MODES) +def test_explicit_fast_loader_failure_preserves_existing_384_store(monkeypatch, tmp_path, mode): + db = tmp_path / "existing-fast.db" + seed_vector_dimensions(db, [384]) + before = hashlib.sha256(db.read_bytes()).hexdigest() + memory_calls = [] + + def unavailable(): + raise RuntimeError("SYNTHETIC_FAST_UNAVAILABLE") + + def forbidden(*args, **kwargs): + memory_calls.append(True) + raise AssertionError("Memory must not open after explicit loader failure") + + set_mode(monkeypatch, mode) + monkeypatch.setattr(embedders, "FastEmbedEmbedder", unavailable) + monkeypatch.setattr(mcp_server, "Memory", forbidden) + with pytest.raises(RuntimeError, match="FAST_EMBEDDER_UNAVAILABLE"): + mcp_server.Server(str(db)) + assert not memory_calls + assert hashlib.sha256(db.read_bytes()).hexdigest() == before + + +@pytest.mark.parametrize("mode,dimensions,denied", [ + (None, [384], True), ("hash", [384], True), ("fast", [256], True), + ("hash", [256, 384], True), ("fast", [384, 256], True), + (None, [256], False), ("fast", [384], False), +]) +def test_real_stdio_dimension_barrier_precedes_initialize_and_remember(tmp_path, mode, dimensions, denied): + db = tmp_path / "stdio.db" + seed_vector_dimensions(db, dimensions) + before = read_vector_snapshot(db) + process, actual_db, imported, constructed = run_stdio_process(tmp_path, mode) + assert actual_db == db + if denied: + assert process.returncode != 0 + assert "EMBEDDER_DIMENSION_MISMATCH" in process.stderr + assert not process.stdout.strip() + assert read_vector_snapshot(db) == before + else: + assert process.returncode == 0, process.stderr + replies = [json.loads(line) for line in process.stdout.splitlines() if line.strip()] + assert [r["id"] for r in replies] == [1, 2] + assert not replies[1]["result"].get("isError") + assert len(read_vector_snapshot(db)) == len(before) + 1 + assert constructed.exists() is (mode == "fast") + + +@pytest.mark.parametrize("seeded", [False, True]) +def test_real_stdio_fast_import_failure_has_no_ready_response_or_fallback(tmp_path, seeded): + db = tmp_path / "stdio.db" + if seeded: + seed_vector_dimensions(db, [384]) + before = read_vector_snapshot(db) + process, _, _, _ = run_stdio_process(tmp_path, "fast", loader_failure=True) + assert process.returncode != 0 + assert "FAST_EMBEDDER_UNAVAILABLE" in process.stderr + assert not process.stdout.strip() + if seeded: + assert read_vector_snapshot(db) == before + else: + assert not db.exists()