Adaptive per-query fusion, on by default

docs/usage.md

@@ -751,6 +751,8 @@ something feels off.
 | `LILBEE_TITLE_SEARCH` | `false` | Match queries against document titles as a third hybrid-search arm, so a query naming a document by title surfaces its chunks |
 | `LILBEE_TITLE_SEARCH_WEIGHT` | `0.5` | Title arm weight in rank fusion (1.0 = equal voice with the vector and text arms) |
 | `LILBEE_LEXICAL_FUSION_WEIGHT` | `1.0` | BM25 arm weight in rank fusion (1.0 = equal to the vector arm; lower it when a strong embedder should dominate a corpus where the lexical arm adds noise) |
+| `LILBEE_ADAPTIVE_FUSION` | `true` | Scale the BM25 arm's weight per query by how confident the vector arm is (a peaked dense ranking downweights lexical, a flat one keeps it) instead of using a fixed `LILBEE_LEXICAL_FUSION_WEIGHT`. That weight becomes the ceiling the adaptive rule scales down from. Set to `false` to pin the fixed weight |
+| `LILBEE_ADAPTIVE_FUSION_MARGIN` | `0.15` | Vector-similarity margin (top hit minus the field) at which adaptive fusion fully silences the BM25 arm; smaller values downweight lexical more aggressively |
 | `LILBEE_ADAPTIVE_THRESHOLD` | `false` | Widen the distance threshold step by step when too few results pass, on the vector-only fallback path (no effect on the default hybrid search) |
 | `LILBEE_ADAPTIVE_THRESHOLD_STEP` | `0.2` | How much to widen per step when adaptive threshold triggers |
 | `LILBEE_TEMPORAL_FILTERING` | `true` | When the query contains temporal cues ("recent", "last week"), filter results by document date and sort by recency |

src/lilbee/app/settings_map.py

@@ -835,6 +835,18 @@ def get_default(key: str) -> object:
         group=SettingGroup.RETRIEVAL,
         help_text="BM25 arm weight in fusion (1.0 = equal to vector; lower to favor dense)",
     ),
+    "adaptive_fusion": SettingDef(
+        bool,
+        nullable=False,
+        group=SettingGroup.RETRIEVAL,
+        help_text="Scale the BM25 weight per query by vector-arm confidence, not a fixed value",
+    ),
+    "adaptive_fusion_margin": SettingDef(
+        float,
+        nullable=False,
+        group=SettingGroup.RETRIEVAL,
+        help_text="Vector-similarity margin at which adaptive fusion fully silences the BM25 arm",
+    ),
     "history_rewrite": SettingDef(
         bool,
         nullable=False,

src/lilbee/core/config/model.py

@@ -210,6 +210,19 @@ class Config(BaseSettings):
     # the retrieval benchmark, not guessed here.
     lexical_fusion_weight: float = ConfigField(default=1.0, ge=0.0, le=1.0, writable=True)
 
+    # Adaptive fusion: instead of a fixed lexical_fusion_weight, scale the BM25
+    # arm per query by how confident the vector arm is (a peaked dense ranking
+    # downweights lexical, a flat one keeps it). On by default: the retrieval
+    # benchmark found no single fixed weight wins every corpus, and adaptive at
+    # margin 0.15 beat the fixed-weight default on all three BEIR sets tested
+    # (biggest gain on the corpus a fixed lexical arm hurt most). lexical_fusion_
+    # weight is the ceiling the adaptive rule scales down from; adaptive_fusion_
+    # margin is the vector-similarity margin at which the lexical arm is fully
+    # silenced (smaller = more aggressive). Set adaptive_fusion=false to pin the
+    # fixed weight instead.
+    adaptive_fusion: bool = ConfigField(default=True, writable=True)
+    adaptive_fusion_margin: float = ConfigField(default=0.15, ge=0.0, le=2.0, writable=True)
+
     # Chunk count at/above which sync builds an approximate (ANN) vector index
     # so search stays fast at millions of vectors. Below this, search uses exact
     # flat scan (faster and exact for small vaults). 0 disables the ANN index.

src/lilbee/data/store/core.py

@@ -28,7 +28,7 @@
 from lilbee.core.security import validate_path_within
 from lilbee.runtime.lock import LOCK_TIMEOUT, write_lock
 
-from .fusion import fuse_arms, normalized_bm25, vector_similarity
+from .fusion import adaptive_lexical_weight, fuse_arms, normalized_bm25, vector_similarity
 from .lance_helpers import (
     _chunk_type_predicate,
     _has_fts_index,
@@ -712,11 +712,18 @@ def _hybrid_search(
         title_rows: list[SearchChunk] = []
         if self._config.title_search:
             title_rows = self._title_arm(table, query_text, top_k, chunk_type)
+        vector_rows = self._vector_arm(table, query_vector, top_k, chunk_type)
+        lexical_weight = self._config.lexical_fusion_weight
+        if self._config.adaptive_fusion:
+            # Gate the lexical arm per query by how peaked the vector ranking is.
+            lexical_weight = adaptive_lexical_weight(
+                vector_rows, lexical_weight, self._config.adaptive_fusion_margin
+            )
         fused = fuse_arms(
-            self._vector_arm(table, query_vector, top_k, chunk_type),
+            vector_rows,
             self._fts_arm(table, query_text, top_k, chunk_type),
             title_rows,
-            lexical_weight=self._config.lexical_fusion_weight,
+            lexical_weight=lexical_weight,
             title_weight=self._config.title_search_weight,
         )
         fused = _drop_unsupported_far_rows(fused, max_distance)

src/lilbee/data/store/fusion.py

@@ -26,17 +26,52 @@
 
 from __future__ import annotations
 
+from statistics import fmean
+
 from .types import SearchChunk
 
 # Standard RRF smoothing constant (Cormack, Clarke & Buettcher 2009).
 _RRF_K = 60
 
+# Adaptive fusion needs a top hit plus at least one field row to measure a margin.
+_MIN_ROWS_FOR_MARGIN = 2
+
 
 def vector_similarity(distance: float) -> float:
     """Cosine distance to canonical [0, 1] similarity (distance spans [0, 2])."""
     return max(0.0, min(1.0, 1.0 - distance))
 
 
+def adaptive_lexical_weight(
+    vector_rows: list[SearchChunk], base_weight: float, margin_scale: float
+) -> float:
+    """Shrink the lexical arm's weight toward zero when the vector arm is
+    confident about this query.
+
+    A *peaked* vector ranking -- a top hit standing well clear of the field --
+    means the dense embedder already located the answer and the lexical arm
+    mostly adds term-match noise (the FiQA case in the retrieval benchmark). A
+    *flat* ranking means dense is unsure and BM25's exact-term matching is worth
+    trusting (NFCorpus, SciFact). The confidence signal is the margin between
+    the top similarity and the mean of the rest, divided by *margin_scale*: at or
+    above that margin the lexical arm is fully silenced, at zero margin it keeps
+    its full *base_weight*, and it scales linearly between. Returns *base_weight*
+    unchanged when there is nothing to measure (fewer than two scored rows) or
+    the feature is disabled (*margin_scale* <= 0).
+    """
+    if margin_scale <= 0:
+        return base_weight
+    sims = sorted(
+        (vector_similarity(r.distance) for r in vector_rows if r.distance is not None),
+        reverse=True,
+    )
+    if len(sims) < _MIN_ROWS_FOR_MARGIN:
+        return base_weight
+    margin = max(0.0, sims[0] - fmean(sims[1:]))
+    confidence = min(1.0, margin / margin_scale)
+    return base_weight * (1.0 - confidence)
+
+
 def normalized_bm25(scores: list[float]) -> list[float]:
     """Scale raw BM25 scores against the list maximum, into (0, 1].
 

tests/test_store.py

@@ -644,6 +644,27 @@ def test_hybrid_search_with_fts_index(self, store, test_config):
         scores = [r.score for r in results]
         assert all(0.0 <= s <= 1.0 for s in scores)
 
+    def test_adaptive_fusion_gates_the_hybrid_search(self, store, test_config):
+        """With adaptive_fusion on (the default), hybrid search derives the
+        lexical weight per query from the vector arm instead of the fixed value."""
+        assert test_config.adaptive_fusion is True  # shipped default
+        store.add_chunks(_make_records())
+        store.ensure_fts_index()
+        query_vec = [0.5] * test_config.embedding_dim
+        results = store.search(query_vec, top_k=3, query_text="chunk number")
+        assert len(results) > 0
+        assert all(r.score is not None and 0.0 <= r.score <= 1.0 for r in results)
+
+    def test_fixed_fusion_when_adaptive_disabled(self, store, test_config):
+        """Opting out of adaptive fusion pins the fixed lexical_fusion_weight."""
+        test_config.adaptive_fusion = False
+        store.add_chunks(_make_records())
+        store.ensure_fts_index()
+        query_vec = [0.5] * test_config.embedding_dim
+        results = store.search(query_vec, top_k=3, query_text="chunk number")
+        assert len(results) > 0
+        assert all(r.score is not None and 0.0 <= r.score <= 1.0 for r in results)
+
     def test_fallback_to_vector_when_no_query_text(self, store, test_config):
         records = _make_records()
         store.add_chunks(records)

tests/test_store_fusion.py

@@ -3,7 +3,12 @@
 import pytest
 
 from lilbee.data.store import SearchChunk
-from lilbee.data.store.fusion import fuse_arms, normalized_bm25, vector_similarity
+from lilbee.data.store.fusion import (
+    adaptive_lexical_weight,
+    fuse_arms,
+    normalized_bm25,
+    vector_similarity,
+)
 
 
 def _chunk(source, idx, *, distance=None, bm25=None, dim=4):
@@ -68,6 +73,52 @@ def test_lexical_only_row_survives_with_score(self):
         assert lexical_row.score == pytest.approx(0.5)
 
 
+class TestAdaptiveLexicalWeight:
+    """Per-query lexical weight gated by the vector arm's confidence."""
+
+    def test_peaked_dense_silences_lexical(self):
+        # top similarity 1.0 (distance 0), field ~0.3: a wide margin => weight ~0.
+        rows = [_chunk("a.md", 0, distance=0.0)] + [
+            _chunk("b.md", i, distance=0.7) for i in range(1, 5)
+        ]
+        assert adaptive_lexical_weight(rows, 1.0, 0.3) == pytest.approx(0.0)
+
+    def test_flat_dense_keeps_full_weight(self):
+        # every row equally similar: zero margin => the arm keeps base_weight.
+        rows = [_chunk("a.md", i, distance=0.5) for i in range(5)]
+        assert adaptive_lexical_weight(rows, 1.0, 0.3) == pytest.approx(1.0)
+
+    def test_scales_linearly_between(self):
+        # top sim 0.6, field 0.3, margin 0.3, scale 0.6 => confidence 0.5 => half.
+        rows = [_chunk("a.md", 0, distance=0.4)] + [
+            _chunk("b.md", i, distance=0.7) for i in range(1, 4)
+        ]
+        assert adaptive_lexical_weight(rows, 1.0, 0.6) == pytest.approx(0.5)
+
+    def test_respects_base_weight(self):
+        rows = [_chunk("a.md", i, distance=0.5) for i in range(3)]
+        assert adaptive_lexical_weight(rows, 0.5, 0.3) == pytest.approx(0.5)
+
+    def test_too_few_rows_returns_base(self):
+        assert adaptive_lexical_weight([_chunk("a.md", 0, distance=0.1)], 1.0, 0.3) == 1.0
+        assert adaptive_lexical_weight([], 1.0, 0.3) == 1.0
+
+    def test_non_positive_margin_scale_disables(self):
+        rows = [_chunk("a.md", 0, distance=0.0), _chunk("b.md", 1, distance=0.9)]
+        assert adaptive_lexical_weight(rows, 1.0, 0.0) == 1.0
+
+    def test_ignores_rows_without_distance(self):
+        # lexical-only rows carry no distance; they must not enter the signal.
+        rows = [
+            _chunk("a.md", 0, distance=0.0),
+            _chunk("b.md", 1, distance=0.6),
+            _chunk("c.md", 2, bm25=9.0),
+        ]
+        both = adaptive_lexical_weight(rows, 1.0, 0.4)
+        two = adaptive_lexical_weight(rows[:2], 1.0, 0.4)
+        assert both == pytest.approx(two)
+
+
 class TestLexicalFusionWeight:
     """The BM25 arm's fusion weight can be lowered so a strong dense arm dominates."""