From b163f2181afbb1f6ca280bdbc03f91ca7e45412a Mon Sep 17 00:00:00 2001 From: chloeeekim Date: Fri, 24 Jul 2026 13:51:32 +0900 Subject: [PATCH] feat: guard against embedder model mismatch between index and query Record the embedder model_id at index time; on query, fall back to lexical (one-time stderr warning) when the model differs. Only the vector dim was checked before, so a same-dim model swap returned silently-wrong results. --- README.md | 2 +- src/fridai/cli.py | 3 +++ src/fridai/core/search.py | 25 +++++++++++++++++++++++++ src/fridai/core/sources/notes.py | 3 +++ src/fridai/core/store.py | 11 +++++++++++ tests/test_search.py | 24 ++++++++++++++++++++++++ tests/test_store.py | 5 +++++ 7 files changed, 72 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b0ad20c..d79a872 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ enable it with `FRIDAI_REDACT_ENTROPY=1`. | `FRIDAI_CODEX_SESSIONS` | `~/.codex/sessions` | Codex CLI session location. | | `FRIDAI_GEMINI_SESSIONS` | `~/.gemini/tmp` | Gemini CLI session location. | | `FRIDAI_EMBED_BACKEND` | auto | `none` disables embeddings (lexical only). | -| `FRIDAI_FASTEMBED_MODEL` | `nomic-ai/nomic-embed-text-v1.5` | fastembed model name. | +| `FRIDAI_FASTEMBED_MODEL` | `nomic-ai/nomic-embed-text-v1.5` | fastembed model name. Changing it needs a full `fridai index --reindex --source all`; set it the same for indexing and querying (incl. the MCP server) or recall falls back to lexical. | | `FRIDAI_REDACT_ENTROPY` | off | `1` enables the high-entropy secret heuristic. | | `FRIDAI_WORK_PENALTY` | `8` | How far to demote pure question turns in ranking. `0` disables. | | `FRIDAI_COMMIT_WINDOW_MIN` | `180` | Minutes window for matching a question to its resulting commit. | diff --git a/src/fridai/cli.py b/src/fridai/cli.py index d282e7c..653bbef 100644 --- a/src/fridai/cli.py +++ b/src/fridai/cli.py @@ -44,6 +44,9 @@ def _run_index(store, source, path, embedder, *, reindex=False, prune=True) -> d """Run one indexing pass for the selected source(s). Returns per-source result dicts. Incremental (mtime/hash state), so repeated passes are cheap — used by both one-shot and --watch.""" out: dict = {} + _mid = getattr(embedder, "model_id", None) # record which model built these vectors + if _mid: + store.set_embedder_id(_mid) if source in ("agent", "all"): out["agent"] = agent_recall.index_all(store, embedder=embedder, reindex=reindex) if source in ("code", "all"): diff --git a/src/fridai/core/search.py b/src/fridai/core/search.py index 8e68306..c8149b6 100644 --- a/src/fridai/core/search.py +++ b/src/fridai/core/search.py @@ -4,10 +4,33 @@ """ from __future__ import annotations +import sys + from . import config from .models import Document, SearchHit from .store import Store +_warned_mismatch: set = set() + + +def _embedder_matches(store: Store, embedder) -> bool: + """True unless the index was built with a different embedder than the query one. + + Only the vector dimension is enforced deeper in the store; two same-dim models would + otherwise return silently-wrong results. On mismatch we warn once (stderr — never stdout, + which is the MCP protocol channel) and let the caller fall back to lexical search.""" + stored = store.get_embedder_id() + current = getattr(embedder, "model_id", None) + if not stored or not current or stored == current: + return True + key = (stored, current) + if key not in _warned_mismatch: + _warned_mismatch.add(key) + print(f"fridai: index was built with embedder '{stored}' but the current embedder is " + f"'{current}'; falling back to lexical search. Re-run `fridai index --reindex " + f"--source all` or set FRIDAI_FASTEMBED_MODEL to match.", file=sys.stderr) + return False + def citation(doc: Document) -> str: """Human-readable source string per document type.""" @@ -120,6 +143,8 @@ def retrieve(store: Store, query: str, k: int = 5, *, embedder=None, repo=None, source_type=None, since=None) -> list[SearchHit]: """Hybrid (BM25+vector RRF) if embedder given, else lexical. Work-signal rerank + dedup, then top-k.""" pool = max(k * 3, 15) # retrieve generously, then trim after rerank/dedup + if embedder is not None and not _embedder_matches(store, embedder): + embedder = None # index built with a different model -> avoid wrong-vector hits if embedder is not None: hits = hybrid_retrieve(store, query, pool, embedder=embedder, repo=repo, source_type=source_type, since=since) diff --git a/src/fridai/core/sources/notes.py b/src/fridai/core/sources/notes.py index b256f7a..e386535 100644 --- a/src/fridai/core/sources/notes.py +++ b/src/fridai/core/sources/notes.py @@ -46,5 +46,8 @@ def add_note(store, text: str, *, repo: str | None = None, ) if embedder: embeddings.embed_documents([doc], embedder) + _mid = getattr(embedder, "model_id", None) # record which model built this vector + if _mid: + store.set_embedder_id(_mid) store.upsert([doc]) return doc diff --git a/src/fridai/core/store.py b/src/fridai/core/store.py index 79afd10..2a67b24 100644 --- a/src/fridai/core/store.py +++ b/src/fridai/core/store.py @@ -322,5 +322,16 @@ def delete_state(self, key: str) -> None: self.con.execute("DELETE FROM index_state WHERE key=?", (key,)) self.con.commit() + # ── embedder identity (index/query consistency) ── + # Records which embedder built the vectors so queries can detect a mismatch + # (e.g. FRIDAI_FASTEMBED_MODEL differs between indexing and querying processes). + _EMBEDDER_KEY = "meta:embedder_model_id" + + def get_embedder_id(self) -> str | None: + return self.get_state(self._EMBEDDER_KEY) + + def set_embedder_id(self, model_id: str) -> None: + self.set_state(self._EMBEDDER_KEY, model_id) + def close(self): self.con.close() diff --git a/tests/test_search.py b/tests/test_search.py index 453a87e..62f4a14 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -111,6 +111,30 @@ def embed(self, q): self.assertTrue(hits) # 0 lexical hits, recalled via vector self.assertEqual(hits[0].document.id, "v1") + def test_retrieve_uses_vector_when_model_matches(self): + self.s.upsert([Document(id="v1", source_type="code", repo="r", path="p", + title="t", text="문서 내용", embedding=[1.0, 0.0])]) + self.s.set_embedder_id("fastembed:A") + + class FakeEmbedder: + model_id = "fastembed:A" + def embed(self, q): + return [1.0, 0.0] + hits = search.retrieve(self.s, "무관한단어", k=3, embedder=FakeEmbedder()) + self.assertEqual(hits[0].document.id, "v1") # matching model -> vector recall works + + def test_retrieve_falls_back_to_lexical_on_model_mismatch(self): + self.s.upsert([Document(id="v1", source_type="code", repo="r", path="p", + title="t", text="문서 내용", embedding=[1.0, 0.0])]) + self.s.set_embedder_id("fastembed:A") # index built with model A + + class FakeEmbedder: + model_id = "fastembed:B" # querying with a different model + def embed(self, q): + return [1.0, 0.0] + hits = search.retrieve(self.s, "무관한단어", k=3, embedder=FakeEmbedder()) + self.assertFalse(hits) # mismatch -> lexical only -> no wrong-vector hit + if __name__ == "__main__": unittest.main() diff --git a/tests/test_store.py b/tests/test_store.py index e6be395..87314c2 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -143,6 +143,11 @@ def test_index_state_roundtrip(self): self.s.set_state("k", "v2") self.assertEqual(self.s.get_state("k"), "v2") + def test_embedder_id_roundtrip(self): + self.assertIsNone(self.s.get_embedder_id()) + self.s.set_embedder_id("fastembed:model-a") + self.assertEqual(self.s.get_embedder_id(), "fastembed:model-a") + def test_delete_state(self): self.s.set_state("k", "v") self.s.delete_state("k")