Make document titles searchable in hybrid retrieval

docs/usage.md

@@ -748,6 +748,9 @@ something feels off.
 | `LILBEE_RERANK_BLEND` | `true` | Blend reranker scores with retrieval fusion; off = the reranker's own ordering stands |
 | `LILBEE_HYDE` | `false` | Enable Hypothetical Document Embeddings: an LLM drafts a hypothetical answer, that's embedded, and results are merged with the original query's. Adds ~500 ms per query; helps on vague questions |
 | `LILBEE_HYDE_WEIGHT` | `0.7` | How much to trust HyDE results relative to the direct query (0.0-1.0) |
+| `LILBEE_TITLE_SEARCH` | `false` | Match queries against document titles as a third hybrid-search arm, so a query naming a document by title surfaces its chunks |
+| `LILBEE_TITLE_SEARCH_WEIGHT` | `0.5` | Title arm weight in rank fusion (1.0 = equal voice with the vector and text arms) |
+| `LILBEE_LEXICAL_FUSION_WEIGHT` | `1.0` | BM25 arm weight in rank fusion (1.0 = equal to the vector arm; lower it when a strong embedder should dominate a corpus where the lexical arm adds noise) |
 | `LILBEE_ADAPTIVE_THRESHOLD` | `false` | Widen the distance threshold step by step when too few results pass, on the vector-only fallback path (no effect on the default hybrid search) |
 | `LILBEE_ADAPTIVE_THRESHOLD_STEP` | `0.2` | How much to widen per step when adaptive threshold triggers |
 | `LILBEE_TEMPORAL_FILTERING` | `true` | When the query contains temporal cues ("recent", "last week"), filter results by document date and sort by recency |

src/lilbee/app/settings_map.py

@@ -817,6 +817,24 @@ def get_default(key: str) -> object:
         group=SettingGroup.RETRIEVAL,
         help_text="Candidate-pool multiplier over top_k before reranking",
     ),
+    "title_search": SettingDef(
+        bool,
+        nullable=False,
+        group=SettingGroup.RETRIEVAL,
+        help_text="Match queries against document titles as a third hybrid-search arm",
+    ),
+    "title_search_weight": SettingDef(
+        float,
+        nullable=False,
+        group=SettingGroup.RETRIEVAL,
+        help_text="Title arm weight in rank fusion (1.0 = equal voice with the other arms)",
+    ),
+    "lexical_fusion_weight": SettingDef(
+        float,
+        nullable=False,
+        group=SettingGroup.RETRIEVAL,
+        help_text="BM25 arm weight in fusion (1.0 = equal to vector; lower to favor dense)",
+    ),
     "history_rewrite": SettingDef(
         bool,
         nullable=False,

src/lilbee/core/config/model.py

@@ -194,6 +194,22 @@ class Config(BaseSettings):
     # fusion arms stay exactly top_k deep.
     candidate_multiplier: int = ConfigField(default=3, ge=1, writable=True)
 
+    # Third lexical arm in hybrid search: BM25 over document titles, fused with
+    # the vector and chunk arms so a query naming a document by title surfaces
+    # its chunks. Off by default until the eval harness measures it.
+    title_search: bool = ConfigField(default=False, writable=True)
+
+    # Title arm weight relative to a full arm in rank fusion (1.0 = equal voice
+    # with the vector and chunk arms).
+    title_search_weight: float = ConfigField(default=0.5, ge=0.0, le=1.0, writable=True)
+
+    # Lexical (BM25) arm weight relative to the vector arm in rank fusion.
+    # 1.0 keeps the two arms equal (the historical behaviour); lowering it lets
+    # a strong dense embedder dominate on corpora where the lexical arm adds
+    # noise rather than signal. The right value is corpus-dependent and set by
+    # the retrieval benchmark, not guessed here.
+    lexical_fusion_weight: float = ConfigField(default=1.0, ge=0.0, le=1.0, writable=True)
+
     # Chunk count at/above which sync builds an approximate (ANN) vector index
     # so search stays fast at millions of vectors. Below this, search uses exact
     # flat scan (faster and exact for small vaults). 0 disables the ANN index.

src/lilbee/data/export.py

@@ -10,7 +10,8 @@
 from typing import TYPE_CHECKING, cast
 
 from lilbee.data.ingest.extract import chunk_and_embed_pages
-from lilbee.data.store import ChunkWrite, PageTextRecord, SourceType
+from lilbee.data.ingest.title import derive_title
+from lilbee.data.store import ChunkWrite, PageTextRecord, SourceMeta, SourceType
 from lilbee.runtime.progress import DetailedProgressCallback, noop_callback
 
 if TYPE_CHECKING:
@@ -225,6 +226,11 @@ async def import_dataset(
         content_type = source_rows[0]["content_type"] or "text"
         page_texts = [(r["page"], r["text"]) for r in source_rows]
         chunks = await chunk_and_embed_pages(page_texts, name, content_type, on_progress)
+        # Imports carry no extraction metadata; the stem-derived title keeps
+        # imported chunks visible to the title search arm.
+        title = derive_title(name)
+        for chunk in chunks:
+            chunk["title"] = title
         # One locked transaction (cleanup + chunks + page texts + source row) so a
         # failure can't leave the source with its old rows deleted and no new ones;
         # the embedding-dim check inside runs before the cleanup delete.
@@ -238,6 +244,7 @@ async def import_dataset(
                     needs_cleanup=True,
                     page_texts=[dict(r) for r in source_rows],
                     source_type=SourceType.IMPORTED,
+                    meta=SourceMeta(title=title),
                 )
             ],
         )

src/lilbee/data/ingest/extract.py

@@ -22,6 +22,7 @@
 from lilbee.data.ingest.discovery import file_hash
 from lilbee.data.ingest.ocr_cache import load_ocr_pages, ocr_cache_key, store_ocr_pages
 from lilbee.data.ingest.offload import to_ingest_thread
+from lilbee.data.ingest.title import source_meta_from_extraction
 from lilbee.data.ingest.types import (
     IMAGE_CONTENT_TYPE,
     MARKDOWN_OUTPUT,
@@ -31,7 +32,7 @@
     ChunkRecord,
     ExtractMode,
 )
-from lilbee.data.store import ChunkType, PageTextRecord
+from lilbee.data.store import ChunkType, PageTextRecord, SourceMeta
 from lilbee.runtime.cancellation import TaskCancelledError
 from lilbee.runtime.cpu import cpu_quota
 from lilbee.runtime.progress import (
@@ -549,11 +550,14 @@ async def ingest_document(
     quiet: bool = False,
     on_progress: DetailedProgressCallback = noop_callback,
     page_texts_out: list[PageTextRecord] | None = None,
+    meta_out: list[SourceMeta] | None = None,
 ) -> list[ChunkRecord]:
     """Extract and chunk a document, embed, return records.
 
     Vision OCR is controlled by ``cfg.enable_ocr`` (see ``_should_run_ocr``).
     When ``page_texts_out`` is given, per-page text is appended for export.
+    When ``meta_out`` is given, the document's extraction metadata (title,
+    authors, creation date) is appended for the source row.
     """
     # An image carries no text layer; route it straight to OCR (vision or Tesseract)
     # instead of a no-op kreuzberg markdown extract that yields nothing for a scan.
@@ -567,6 +571,11 @@ async def ingest_document(
     config = extraction_config(content_type_to_mode(content_type))
     result = await to_ingest_thread(extract_file_sync, str(path), config=config)
 
+    # Captured before the scanned-PDF fallback: a scan's PDF metadata (title,
+    # authors) survives even when its text layer is empty.
+    if meta_out is not None:
+        meta_out.append(source_meta_from_extraction(result.metadata or {}, source_name))
+
     if content_type == PDF_CONTENT_TYPE and not _has_meaningful_text(result):
         return await _handle_scanned_pdf_fallback(
             path,

src/lilbee/data/ingest/pipeline.py

@@ -41,6 +41,7 @@
     write_skip_markers,
     write_skip_reasons,
 )
+from lilbee.data.ingest.title import derive_title
 from lilbee.data.ingest.types import (
     ChunkRecord,
     FileChangePlan,
@@ -53,6 +54,7 @@
     ChunkWrite,
     ConceptRecords,
     PageTextRecord,
+    SourceMeta,
     SourceRecord,
     SourceStat,
     SourceStatBackfill,
@@ -198,30 +200,42 @@ async def _produce_records(
     quiet: bool = False,
     on_progress: DetailedProgressCallback = noop_callback,
     page_texts_out: list[PageTextRecord] | None = None,
+    meta_out: list[SourceMeta] | None = None,
 ) -> list[ChunkRecord]:
     """Extract, chunk, and embed a single file into store-ready records.
 
     The LanceDB write is deferred: records are returned to the caller and written
     in a batched flush (see :func:`_flush_writes`), so bulk ingest pays one
     write-lock acquisition per batch instead of one per file. The per-page text
-    dataset rows land in ``page_texts_out`` and are written by the same flush.
+    dataset rows land in ``page_texts_out`` and are written by the same flush;
+    the document's metadata (extraction-provided when available, stem-derived
+    title otherwise) lands in ``meta_out`` and stamps every record's ``title``.
     """
     records: list[ChunkRecord]
     page_texts: list[PageTextRecord] = page_texts_out if page_texts_out is not None else []
+    meta = SourceMeta(title=derive_title(source_name))
     if content_type == "code":
         records = await to_ingest_thread(ingest_code_sync, path, source_name, on_progress)
     elif path.suffix.lower() == ".md":
         records = await ingest_markdown(path, source_name, on_progress, page_texts_out=page_texts)
     else:
+        extracted_meta: list[SourceMeta] = []
         records = await ingest_document(
             path,
             source_name,
             content_type,
             quiet=quiet,
             on_progress=on_progress,
             page_texts_out=page_texts,
+            meta_out=extracted_meta,
         )
+        if extracted_meta:
+            meta = extracted_meta[0]
 
+    for record in records:
+        record["title"] = meta.title
+    if meta_out is not None:
+        meta_out.append(meta)
     return records
 
 
@@ -680,13 +694,15 @@ async def _process_one(entry: FileToProcess, file_index: int) -> _IngestResult:
                 # transaction as the new write (see _flush_writes), so cleanup is
                 # carried on the result rather than run eagerly here.
                 page_texts: list[PageTextRecord] = []
+                meta_out: list[SourceMeta] = []
                 records = await _produce_records(
                     entry.path,
                     name,
                     entry.content_type,
                     quiet=quiet,
                     on_progress=on_progress,
                     page_texts_out=page_texts,
+                    meta_out=meta_out,
                 )
                 concept_records = await _build_concept_records(records, name)
                 entity_rows = await _build_entity_records(records, name)
@@ -707,6 +723,7 @@ async def _process_one(entry: FileToProcess, file_index: int) -> _IngestResult:
                     stat=entry.stat,
                     concept_records=concept_records,
                     entity_rows=entity_rows,
+                    meta=meta_out[0] if meta_out else None,
                 )
             except (asyncio.CancelledError, TaskCancelledError) as exc:
                 # TaskCancelledError is the TUI's cooperative cancel signal raised
@@ -1031,6 +1048,7 @@ def _flush_batch(buffer: list[_IngestResult]) -> None:
             needs_cleanup=r.needs_cleanup,
             stat=r.stat,
             page_texts=cast(list[dict], r.page_texts or []),
+            meta=r.meta,
         )
         for r in buffer
     ]

src/lilbee/data/ingest/title.py

@@ -0,0 +1,38 @@
+"""Document title and source-level metadata derivation for ingest."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from pathlib import PurePath
+from typing import Any
+
+from lilbee.data.store import SourceMeta
+
+# Filename-stem separators flattened to spaces when no extracted title exists.
+_STEM_SEPARATOR_RE = re.compile(r"[_\-\s]+")
+
+
+def derive_title(source_name: str, metadata_title: str | None = None) -> str:
+    """Human-readable document title: the extracted title, else the cleaned filename stem.
+
+    The stem cleanup flattens underscore/hyphen separators to spaces so BM25
+    tokenizes ``survey_214.pdf`` into the same terms a query would use.
+    """
+    if metadata_title and metadata_title.strip():
+        return metadata_title.strip()
+    return _STEM_SEPARATOR_RE.sub(" ", PurePath(source_name).stem).strip()
+
+
+def source_meta_from_extraction(metadata: Mapping[str, Any], source_name: str) -> SourceMeta:
+    """Fold kreuzberg extraction metadata into a :class:`SourceMeta`.
+
+    The title falls back to the filename stem; authors and creation date stay
+    empty (persisted NULL) when the extractor reports none.
+    """
+    authors = metadata.get("authors") or []
+    return SourceMeta(
+        title=derive_title(source_name, metadata.get("title")),
+        authors=", ".join(a for a in authors if a),
+        created_at=str(metadata.get("created_at") or ""),
+    )

src/lilbee/data/ingest/types.py

@@ -5,14 +5,15 @@
 from dataclasses import dataclass
 from enum import StrEnum
 from pathlib import Path
-from typing import NamedTuple, TypedDict
+from typing import NamedTuple, NotRequired, TypedDict
 
 from pydantic import BaseModel
 
 from lilbee.data.store import (
     ChunkType,
     ConceptRecords,
     PageTextRecord,
+    SourceMeta,
     SourceStat,
     SourceStatBackfill,
 )
@@ -72,6 +73,9 @@ class ChunkRecord(TypedDict):
     chunk: str
     chunk_index: int
     vector: list[float]
+    # Stamped once per document by the pipeline (see _produce_records) rather
+    # than set by every producer, so the derivation lives in one place.
+    title: NotRequired[str]
 
 
 class SyncResult(BaseModel):
@@ -124,7 +128,8 @@ class _IngestResult:
     travels with the records so the flush can delete the source's old chunks in
     the same transaction. ``page_texts`` carries the per-page text dataset rows
     and ``concept_records`` the file's concept-table rows, and ``entity_rows``
-    the file's typed-entity rows, all written by the same flush.
+    the file's typed-entity rows, all written by the same flush. ``meta``
+    carries the document's extraction-time metadata for the source row.
     """
 
     name: str
@@ -138,6 +143,7 @@ class _IngestResult:
     stat: SourceStat | None = None
     concept_records: ConceptRecords | None = None
     entity_rows: list[dict] | None = None
+    meta: SourceMeta | None = None
 
 
 # Extension → content_type string for document formats handled by kreuzberg.

src/lilbee/data/store/__init__.py

@@ -28,6 +28,7 @@
     RemoveResult,
     SearchChunk,
     SearchScope,
+    SourceMeta,
     SourceRecord,
     SourceStat,
     SourceStatBackfill,
@@ -53,6 +54,7 @@
     "RemoveResult",
     "SearchChunk",
     "SearchScope",
+    "SourceMeta",
     "SourceRecord",
     "SourceStat",
     "SourceStatBackfill",