Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ enable it with `FRIDAI_REDACT_ENTROPY=1`.
| `FRIDAI_CODEX_SESSIONS` | `~/.codex/sessions` | Codex CLI session location. |
| `FRIDAI_GEMINI_SESSIONS` | `~/.gemini/tmp` | Gemini CLI session location. |
| `FRIDAI_EMBED_BACKEND` | auto | `none` disables embeddings (lexical only). |
| `FRIDAI_FASTEMBED_MODEL` | `nomic-ai/nomic-embed-text-v1.5` | fastembed model name. |
| `FRIDAI_FASTEMBED_MODEL` | `nomic-ai/nomic-embed-text-v1.5` | fastembed model name. Changing it needs a full `fridai index --reindex --source all`; set it the same for indexing and querying (incl. the MCP server) or recall falls back to lexical. |
| `FRIDAI_REDACT_ENTROPY` | off | `1` enables the high-entropy secret heuristic. |
| `FRIDAI_WORK_PENALTY` | `8` | How far to demote pure question turns in ranking. `0` disables. |
| `FRIDAI_COMMIT_WINDOW_MIN` | `180` | Minutes window for matching a question to its resulting commit. |
Expand Down
3 changes: 3 additions & 0 deletions src/fridai/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ def _run_index(store, source, path, embedder, *, reindex=False, prune=True) -> d
"""Run one indexing pass for the selected source(s). Returns per-source result dicts.
Incremental (mtime/hash state), so repeated passes are cheap — used by both one-shot and --watch."""
out: dict = {}
_mid = getattr(embedder, "model_id", None) # record which model built these vectors
if _mid:
store.set_embedder_id(_mid)
if source in ("agent", "all"):
out["agent"] = agent_recall.index_all(store, embedder=embedder, reindex=reindex)
if source in ("code", "all"):
Expand Down
25 changes: 25 additions & 0 deletions src/fridai/core/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,33 @@
"""
from __future__ import annotations

import sys

from . import config
from .models import Document, SearchHit
from .store import Store

_warned_mismatch: set = set()


def _embedder_matches(store: Store, embedder) -> bool:
"""True unless the index was built with a different embedder than the query one.

Only the vector dimension is enforced deeper in the store; two same-dim models would
otherwise return silently-wrong results. On mismatch we warn once (stderr — never stdout,
which is the MCP protocol channel) and let the caller fall back to lexical search."""
stored = store.get_embedder_id()
current = getattr(embedder, "model_id", None)
if not stored or not current or stored == current:
return True
key = (stored, current)
if key not in _warned_mismatch:
_warned_mismatch.add(key)
print(f"fridai: index was built with embedder '{stored}' but the current embedder is "
f"'{current}'; falling back to lexical search. Re-run `fridai index --reindex "
f"--source all` or set FRIDAI_FASTEMBED_MODEL to match.", file=sys.stderr)
return False


def citation(doc: Document) -> str:
"""Human-readable source string per document type."""
Expand Down Expand Up @@ -120,6 +143,8 @@ def retrieve(store: Store, query: str, k: int = 5, *, embedder=None,
repo=None, source_type=None, since=None) -> list[SearchHit]:
"""Hybrid (BM25+vector RRF) if embedder given, else lexical. Work-signal rerank + dedup, then top-k."""
pool = max(k * 3, 15) # retrieve generously, then trim after rerank/dedup
if embedder is not None and not _embedder_matches(store, embedder):
embedder = None # index built with a different model -> avoid wrong-vector hits
if embedder is not None:
hits = hybrid_retrieve(store, query, pool, embedder=embedder,
repo=repo, source_type=source_type, since=since)
Expand Down
3 changes: 3 additions & 0 deletions src/fridai/core/sources/notes.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,5 +46,8 @@ def add_note(store, text: str, *, repo: str | None = None,
)
if embedder:
embeddings.embed_documents([doc], embedder)
_mid = getattr(embedder, "model_id", None) # record which model built this vector
if _mid:
store.set_embedder_id(_mid)
store.upsert([doc])
return doc
11 changes: 11 additions & 0 deletions src/fridai/core/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,5 +322,16 @@ def delete_state(self, key: str) -> None:
self.con.execute("DELETE FROM index_state WHERE key=?", (key,))
self.con.commit()

# ── embedder identity (index/query consistency) ──
# Records which embedder built the vectors so queries can detect a mismatch
# (e.g. FRIDAI_FASTEMBED_MODEL differs between indexing and querying processes).
_EMBEDDER_KEY = "meta:embedder_model_id"

def get_embedder_id(self) -> str | None:
return self.get_state(self._EMBEDDER_KEY)

def set_embedder_id(self, model_id: str) -> None:
self.set_state(self._EMBEDDER_KEY, model_id)

def close(self):
self.con.close()
24 changes: 24 additions & 0 deletions tests/test_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,30 @@ def embed(self, q):
self.assertTrue(hits) # 0 lexical hits, recalled via vector
self.assertEqual(hits[0].document.id, "v1")

def test_retrieve_uses_vector_when_model_matches(self):
self.s.upsert([Document(id="v1", source_type="code", repo="r", path="p",
title="t", text="문서 내용", embedding=[1.0, 0.0])])
self.s.set_embedder_id("fastembed:A")

class FakeEmbedder:
model_id = "fastembed:A"
def embed(self, q):
return [1.0, 0.0]
hits = search.retrieve(self.s, "무관한단어", k=3, embedder=FakeEmbedder())
self.assertEqual(hits[0].document.id, "v1") # matching model -> vector recall works

def test_retrieve_falls_back_to_lexical_on_model_mismatch(self):
self.s.upsert([Document(id="v1", source_type="code", repo="r", path="p",
title="t", text="문서 내용", embedding=[1.0, 0.0])])
self.s.set_embedder_id("fastembed:A") # index built with model A

class FakeEmbedder:
model_id = "fastembed:B" # querying with a different model
def embed(self, q):
return [1.0, 0.0]
hits = search.retrieve(self.s, "무관한단어", k=3, embedder=FakeEmbedder())
self.assertFalse(hits) # mismatch -> lexical only -> no wrong-vector hit


if __name__ == "__main__":
unittest.main()
5 changes: 5 additions & 0 deletions tests/test_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,11 @@ def test_index_state_roundtrip(self):
self.s.set_state("k", "v2")
self.assertEqual(self.s.get_state("k"), "v2")

def test_embedder_id_roundtrip(self):
self.assertIsNone(self.s.get_embedder_id())
self.s.set_embedder_id("fastembed:model-a")
self.assertEqual(self.s.get_embedder_id(), "fastembed:model-a")

def test_delete_state(self):
self.s.set_state("k", "v")
self.s.delete_state("k")
Expand Down
Loading