From 0235ad8e41db345587f3a285881f5b24413fe910 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Wed, 24 Jun 2026 13:58:40 -0400 Subject: [PATCH 01/77] feat: integrate kreuzberg 5.x Migrate from kreuzberg 4.x to 5.x. Replace lilbee's in-process PDF rasterization, OCR orchestration, and per-page OCR cache with kreuzberg's extraction pipeline, exposing lilbee's vision model as a custom kreuzberg OCR backend. Single extraction path; per-page citations preserved. Fetch the vision mmproj on pull when the main GGUF is already installed (bb-7yd). Start the fleet reliably and accept kreuzberg's JSON-string OcrConfig callback (rc.35). --- pyproject.toml | 3 +- src/lilbee/app/models.py | 17 + src/lilbee/app/services.py | 29 + src/lilbee/app/settings.py | 6 + src/lilbee/catalog/__init__.py | 2 + src/lilbee/catalog/download.py | 4 +- src/lilbee/data/chunk.py | 16 +- src/lilbee/data/ingest/extract.py | 488 +++--------- src/lilbee/data/ingest/ocr_cache.py | 90 --- src/lilbee/data/ingest/types.py | 18 +- src/lilbee/data/ingest/vision_ocr_backend.py | 177 +++++ src/lilbee/providers/base.py | 20 +- src/lilbee/providers/fleet/provider.py | 149 +--- src/lilbee/providers/fleet/swap_manager.py | 1 + src/lilbee/providers/routing_provider.py | 30 +- src/lilbee/providers/sdk_llm_provider.py | 19 - src/lilbee/runtime/progress/types.py | 8 +- src/lilbee/server/handlers/models.py | 2 + src/lilbee/vision.py | 52 +- tests/integration/test_pdf_integration.py | 53 +- tests/server/test_handlers.py | 3 +- tests/test_chunker.py | 10 +- tests/test_cli_model.py | 27 + tests/test_dynamic_settings_eviction.py | 13 +- tests/test_fleet_provider.py | 174 ----- tests/test_formats.py | 42 +- tests/test_ingest.py | 782 ++++--------------- tests/test_ocr_cache.py | 125 --- tests/test_providers.py | 59 -- tests/test_services.py | 66 ++ tests/test_vision.py | 161 +--- tests/test_vision_ocr_backend.py | 129 +++ uv.lock | 12 +- 33 files changed, 804 insertions(+), 1983 deletions(-) delete mode 100644 src/lilbee/data/ingest/ocr_cache.py create mode 100644 src/lilbee/data/ingest/vision_ocr_backend.py delete mode 100644 tests/test_ocr_cache.py create mode 100644 tests/test_vision_ocr_backend.py diff --git a/pyproject.toml b/pyproject.toml index e984d2b2d..3c3d286df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ classifiers = [ dependencies = [ "lancedb", - "kreuzberg>=4.9.1,<5", + "kreuzberg>=5.0.0rc35", "filelock", "tree-sitter-language-pack>=1.8.0,<2.0", "typer>=0.12", @@ -92,6 +92,7 @@ packages = ["src/lilbee"] # the index instead -- this source override is workspace metadata, not published deps. [tool.uv.sources] lilbee-engine = { path = "packaging/engine-wheel" } +kreuzberg = { path = "/Users/tobias/projects/kreuzberg/.worktrees/rc35/packages/python/dist/kreuzberg-5.0.0rc35-cp310-abi3-macosx_11_0_arm64.whl" } # LOCAL DEV ONLY (uncommitted) [dependency-groups] dev = [ diff --git a/src/lilbee/app/models.py b/src/lilbee/app/models.py index 8cb992be0..8df7fce8f 100644 --- a/src/lilbee/app/models.py +++ b/src/lilbee/app/models.py @@ -318,6 +318,19 @@ def show_model_data(ref: str) -> ShowModelResult: ) +def _ensure_vision_projector(ref: str) -> None: + """Fetch a vision model's mmproj projector when a cached install lacks it (bb-7yd). + + No-op for non-vision refs. ``download_mmproj`` is idempotent against the HF cache. + """ + from lilbee.catalog import download_mmproj, resolve_pull_target + from lilbee.catalog.types import ModelTask + + entry = resolve_pull_target(ref) + if entry is not None and entry.task is ModelTask.VISION: + download_mmproj(entry) + + def pull_model_data( ref: str, source: ModelSource, @@ -338,6 +351,10 @@ def pull_model_data( manager = get_services().model_manager if manager.is_installed(ref, source): + # A cached vision install may carry the main GGUF but not its mmproj projector + # (bb-7yd); without it llama-server can't serve OCR, so ensure it before + # reporting already-installed. + _ensure_vision_projector(ref) return PullResult(model=ref, source=source.value, status=PullStatus.ALREADY_INSTALLED) bytes_cb = make_download_callback(on_update) if on_update is not None else None diff --git a/src/lilbee/app/services.py b/src/lilbee/app/services.py index 075da261c..4b7296737 100644 --- a/src/lilbee/app/services.py +++ b/src/lilbee/app/services.py @@ -185,6 +185,8 @@ def get_services() -> Services: # are skipped, so a setup with only chat + embed never spawns rerank or # vision. Set ``cfg.worker_pool_eager_start = false`` for headless scripts # where mount time matters more than first-call latency. + sync_vision_ocr_backend(provider) + # Eager start is the default: pay the spawn cost per role server at TUI mount if cfg.worker_pool_eager_start: from contextlib import suppress @@ -193,6 +195,33 @@ def get_services() -> Services: return _svc +def sync_vision_ocr_backend(provider: LLMProvider) -> None: + """Register or unregister lilbee's vision model as kreuzberg's OCR backend. + + Driven by ``cfg.vision_model``: registered while a model is set, removed when + cleared. The backend reads ``cfg.vision_model`` live, so a model swap needs no + re-registration, but it captures ``provider.vision_ocr``, so it must re-bind + whenever the provider is rebuilt (``reset_services``) -- otherwise the global + kreuzberg registry keeps routing OCR to the shut-down provider. + """ + from kreuzberg import list_ocr_backends, register_ocr_backend, unregister_ocr_backend + + from lilbee.core.config import cfg + from lilbee.data.ingest.types import OcrBackendName + from lilbee.data.ingest.vision_ocr_backend import VisionOcrBackend + + registered = OcrBackendName.LILBEE_VISION in list_ocr_backends() + if cfg.vision_model: + # Re-register so the backend always binds to the current provider. + if registered: + unregister_ocr_backend(OcrBackendName.LILBEE_VISION) + register_ocr_backend( + VisionOcrBackend(ocr_fn=provider.vision_ocr, model_ref_fn=lambda: cfg.vision_model) + ) + elif registered: + unregister_ocr_backend(OcrBackendName.LILBEE_VISION) + + def set_services(services: Services | None) -> None: """Replace the cached Services singleton (for testing).""" global _svc diff --git a/src/lilbee/app/settings.py b/src/lilbee/app/settings.py index 8f64f7944..43d5d719d 100644 --- a/src/lilbee/app/settings.py +++ b/src/lilbee/app/settings.py @@ -272,6 +272,12 @@ def _reload_changed_roles(changed_keys: set[str]) -> None: changed_role_fields = changed_keys & MODEL_ROLE_FIELDS for field in changed_role_fields: services.reload_role(MODEL_FIELD_TO_ROLE[field]) + if "vision_model" in changed_role_fields: + # Register/unregister lilbee's kreuzberg OCR backend on any vision-model + # change (REST/MCP/TUI/CLI all funnel here), not just the REST route. + from lilbee.app.services import sync_vision_ocr_backend + + sync_vision_ocr_backend(services.provider) role_agnostic = (changed_keys & LOAD_AFFECTING_KEYS) - MODEL_ROLE_FIELDS if role_agnostic: services.provider.drop_loaded_models_async() diff --git a/src/lilbee/catalog/__init__.py b/src/lilbee/catalog/__init__.py index 6a430fe1e..d9f7a4c1f 100644 --- a/src/lilbee/catalog/__init__.py +++ b/src/lilbee/catalog/__init__.py @@ -8,6 +8,7 @@ from lilbee.catalog.download import ( DownloadConfig, + download_mmproj, download_model, find_mmproj_file, resolve_filename, @@ -70,6 +71,7 @@ "build_adhoc_entry", "clean_display_name", "display_label_for_ref", + "download_mmproj", "download_model", "download_task_name", "enrich_catalog", diff --git a/src/lilbee/catalog/download.py b/src/lilbee/catalog/download.py index ebaac0208..691dfbedf 100644 --- a/src/lilbee/catalog/download.py +++ b/src/lilbee/catalog/download.py @@ -221,11 +221,11 @@ def _finalize_download( if on_complete is not None: on_complete(entry, dest) if entry.task == ModelTask.VISION: - _download_mmproj(entry, on_progress=on_progress) + download_mmproj(entry, on_progress=on_progress) return dest -def _download_mmproj( +def download_mmproj( entry: CatalogModel, *, on_progress: ProgressCallback | None = None, diff --git a/src/lilbee/data/chunk.py b/src/lilbee/data/chunk.py index 3647224e2..1b9f13071 100644 --- a/src/lilbee/data/chunk.py +++ b/src/lilbee/data/chunk.py @@ -38,7 +38,7 @@ def _show_download_progress() -> bool: def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: """Build a kreuzberg ChunkingConfig from the current cfg.""" - from kreuzberg import ChunkingConfig, EmbeddingConfig, EmbeddingModelType + from kreuzberg import ChunkingConfig, EmbeddingConfig max_chars, max_overlap = _char_budget() @@ -46,14 +46,14 @@ def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: return ChunkingConfig( chunker_type=_SEMANTIC_CHUNKER, embedding=EmbeddingConfig( - model=EmbeddingModelType.preset(_SEMANTIC_EMBEDDING_PRESET), + model=_SEMANTIC_EMBEDDING_PRESET, show_download_progress=_show_download_progress(), ), topic_threshold=cfg.topic_threshold, - max_chars=max_chars, - max_overlap=max_overlap, + max_characters=max_chars, + overlap=max_overlap, ) - return ChunkingConfig(max_chars=max_chars, max_overlap=max_overlap) + return ChunkingConfig(max_characters=max_chars, overlap=max_overlap) def chunk_text( @@ -72,10 +72,10 @@ def chunk_text( if heading_context: max_chars, max_overlap = _char_budget() chunking = ChunkingConfig( - max_chars=max_chars, - max_overlap=max_overlap, + max_characters=max_chars, + overlap=max_overlap, chunker_type=_MARKDOWN_CHUNKER, - prepend_heading_context=True, # type: ignore[call-arg] + prepend_heading_context=True, ) else: chunking = build_chunking_config(use_semantic=use_semantic) diff --git a/src/lilbee/data/ingest/extract.py b/src/lilbee/data/ingest/extract.py index b48c58b95..37ec6da9d 100644 --- a/src/lilbee/data/ingest/extract.py +++ b/src/lilbee/data/ingest/extract.py @@ -1,37 +1,29 @@ -"""Document extraction: kreuzberg config, OCR fallback, markdown/document chunking.""" +"""Document extraction: one kreuzberg pass that natively extracts text and OCRs +scanned pages/images through the registered backend; chunk + embed the result.""" from __future__ import annotations import asyncio import contextvars import logging -from collections.abc import Awaitable, Callable, Generator, Sequence +from collections.abc import Generator, Sequence from contextlib import contextmanager -from io import BytesIO from pathlib import Path from typing import TYPE_CHECKING, Any -from PIL import Image, ImageSequence - -if TYPE_CHECKING: - from kreuzberg import ExtractionConfig, ExtractionResult - from lilbee.app.services import get_services from lilbee.core.config import cfg from lilbee.data.chunk import build_chunking_config, chunk_text -from lilbee.data.ingest.discovery import file_hash -from lilbee.data.ingest.ocr_cache import load_ocr_pages, ocr_cache_key, store_ocr_pages from lilbee.data.ingest.types import ( IMAGE_CONTENT_TYPE, MARKDOWN_OUTPUT, - MIN_MEANINGFUL_CHARS, PDF_CONTENT_TYPE, - TESSERACT_BACKEND, ChunkRecord, ExtractMode, + OcrBackendName, ) +from lilbee.data.ingest.vision_ocr_backend import backend_options_for, ocr_request from lilbee.data.store import ChunkType, PageTextRecord -from lilbee.runtime.cancellation import TaskCancelledError from lilbee.runtime.cpu import cpu_quota from lilbee.runtime.progress import ( DetailedProgressCallback, @@ -40,19 +32,21 @@ noop_callback, ) -log = logging.getLogger(__name__) +if TYPE_CHECKING: + from kreuzberg import ExtractionConfig, OcrConfig + # extract_* return the pyo3 result (attribute access), not the public + # ExtractionResult TypedDict (kreuzberg-7ih). + from kreuzberg._kreuzberg import ExtractionResult -def _has_meaningful_text(result: ExtractionResult) -> bool: - """True when extraction yielded real text; content is primary, chunks the fallback.""" - if len(result.content.strip()) > MIN_MEANINGFUL_CHARS: - return True - return sum(len(c.content.strip()) for c in result.chunks or []) > MIN_MEANINGFUL_CHARS +log = logging.getLogger(__name__) def content_type_to_mode(content_type: str) -> ExtractMode: - """Map a content_type to the extraction mode.""" - return ExtractMode.PAGINATED if content_type == PDF_CONTENT_TYPE else ExtractMode.MARKDOWN + """Map a content_type to the extraction mode (paginated for PDFs and images).""" + if content_type in (PDF_CONTENT_TYPE, IMAGE_CONTENT_TYPE): + return ExtractMode.PAGINATED + return ExtractMode.MARKDOWN def _page_text_record(source: str, page: int, text: str, content_type: str) -> PageTextRecord: @@ -60,37 +54,6 @@ def _page_text_record(source: str, page: int, text: str, content_type: str) -> P return PageTextRecord(source=source, page=page, text=text, content_type=content_type) -def extraction_config(mode: ExtractMode) -> ExtractionConfig: - """Build ExtractionConfig for the given extraction mode.""" - from kreuzberg import ConcurrencyConfig, ExtractionConfig, OcrConfig, PageConfig - - chunking = build_chunking_config() - pages = PageConfig(extract_pages=True, insert_page_markers=False) - ocr = OcrConfig(backend=TESSERACT_BACKEND) - # Bound kreuzberg's internal pool to the same CPU budget as the - # pipeline semaphore so the two stop competing for cores. - concurrency = ConcurrencyConfig(max_threads=cpu_quota()) - builders: dict[ExtractMode, Callable[[], ExtractionConfig]] = { - ExtractMode.MARKDOWN: lambda: ExtractionConfig( - chunking=chunking, - output_format=MARKDOWN_OUTPUT, - concurrency=concurrency, - ), - ExtractMode.PAGINATED: lambda: ExtractionConfig( - chunking=chunking, - pages=pages, - concurrency=concurrency, - ), - ExtractMode.PAGINATED_OCR: lambda: ExtractionConfig( - chunking=chunking, - pages=pages, - ocr=ocr, - concurrency=concurrency, - ), - } - return builders[mode]() - - _ocr_enable_override: contextvars.ContextVar[bool | None] = contextvars.ContextVar( "lilbee_ocr_enable_override", default=None ) @@ -103,9 +66,7 @@ def _effective_enable_ocr() -> bool | None: """``cfg.enable_ocr`` unless a per-request OCR override is active. The override is a ContextVar, not a global cfg mutation, so concurrent - ingests on the shared HTTP daemon each see their own setting. The override - also propagates into ``asyncio.to_thread`` workers (``to_thread`` copies the - calling task's context), which is how the timeout reaches the image-OCR call. + ingests on the shared HTTP daemon each see their own setting. """ override = _ocr_enable_override.get() return cfg.enable_ocr if override is None else override @@ -124,8 +85,8 @@ def ocr_override( """Scope per-request OCR settings without mutating the global cfg. A ``None`` argument leaves that setting at its cfg default. Each override is - isolated to the entering context (and any ``to_thread`` work it spawns), so - overlapping ingests never clobber one another's OCR config. + isolated to the entering context, so overlapping ingests never clobber one + another's OCR config. """ tokens: list[tuple[contextvars.ContextVar[Any], contextvars.Token[Any]]] = [] try: @@ -139,235 +100,50 @@ def ocr_override( var.reset(token) -def _should_run_ocr() -> bool: - """Decide whether to attempt vision-based OCR on scanned PDFs. +def _ocr_config(ocr_token: str | None) -> OcrConfig: + """Pick the OCR backend for this extraction. - Uses the effective OCR setting (``cfg.enable_ocr`` or a per-request - override) and ``cfg.vision_model``: - True = force on (requires ``cfg.vision_model`` to be set for a real - vision run; otherwise the caller falls back to Tesseract). - False = force off. - None = auto-detect: run vision OCR when ``cfg.vision_model`` is set. + Mirrors the prior fallback policy: OCR off when ``enable_ocr`` is False; lilbee's + vision backend when a vision model is configured; otherwise kreuzberg's tesseract. + kreuzberg auto-OCRs only the pages that lack a text layer. """ - enable_ocr = _effective_enable_ocr() - if enable_ocr is True: - return True - if enable_ocr is False: - return False - return bool(cfg.vision_model) - + from kreuzberg import OcrConfig -def _record_page_texts( - page_texts: Sequence[tuple[int, str]], - source_name: str, - content_type: str, - page_texts_out: list[PageTextRecord] | None, -) -> None: - """Append OCR page texts to the export accumulator when one is supplied.""" - if page_texts_out is None: - return - page_texts_out.extend( - _page_text_record(source_name, page, text, content_type) for page, text in page_texts - ) - - -async def _vision_ocr_cached( - path: Path, - source_name: str, - content_type: str, - *, - ocr_fn: Callable[[], Awaitable[list[tuple[int, str]]]], - on_progress: DetailedProgressCallback, - page_texts_out: list[PageTextRecord] | None = None, -) -> list[ChunkRecord]: - """Cache-wrapped vision OCR: reuse stored pages, else run *ocr_fn*, then chunk + embed. + if _effective_enable_ocr() is False: + return OcrConfig(enabled=False) + if cfg.vision_model: + options = backend_options_for(ocr_token) if ocr_token else None + return OcrConfig(backend=OcrBackendName.LILBEE_VISION, backend_options=options) + return OcrConfig(backend=OcrBackendName.TESSERACT) - Pool routing amortises the multi-second vision-Llama load across files. - *ocr_fn* returns the OCR'd pages as ``(page_number, text)`` tuples (a PDF page - loop or a single image). Output is cached by file content + model, so a retry - after a downstream failure (chunk/embed/store) reuses the pages, not re-OCR-ing. - """ - # The per-page timeout bounds completeness: a page that exhausts the budget - # yields empty text. Key on it so raising the timeout re-OCRs the file rather - # than serving the earlier, partially-empty cached result for the same content. - key = ocr_cache_key( - file_hash(path), - backend="vision", - model=cfg.vision_model, - extra=f"{cfg.vision_ocr_max_tokens}:{_effective_ocr_timeout()}", - ) - cached = load_ocr_pages(key) - if cached is not None: - _record_page_texts(cached, source_name, content_type, page_texts_out) - return await chunk_and_embed_pages(cached, source_name, content_type, on_progress) - try: - pages = await ocr_fn() - except (asyncio.CancelledError, TaskCancelledError): - # A user cancel (SIGINT / TUI cancel) raised cooperatively through the - # per-page on_progress callback must abort the file, not be logged as an - # OCR failure and swallowed. - raise - except Exception: - log.warning("OCR via vision backend failed for %s.", source_name, exc_info=True) - return [] - store_ocr_pages(key, pages) - _record_page_texts(pages, source_name, content_type, page_texts_out) - return await chunk_and_embed_pages(pages, source_name, content_type, on_progress) +def extraction_config(mode: ExtractMode, *, ocr_token: str | None = None) -> ExtractionConfig: + """Build ExtractionConfig for the given extraction mode.""" + from kreuzberg import ExtractionConfig, PageConfig -async def _vision_ocr_fallback( - path: Path, - source_name: str, - content_type: str, - *, - on_progress: DetailedProgressCallback, - quiet: bool, - page_texts_out: list[PageTextRecord] | None = None, -) -> list[ChunkRecord]: - """Vision OCR a scanned PDF: rasterize + OCR each page through the worker pool.""" - - async def _ocr() -> list[tuple[int, str]]: - pages = await asyncio.to_thread( - get_services().provider.pdf_ocr, - path, - backend="vision", - model=cfg.vision_model, - per_page_timeout_s=_effective_ocr_timeout(), - quiet=quiet, - on_progress=on_progress, + chunking = build_chunking_config() + ocr = _ocr_config(ocr_token) + # Bound batch extraction to the CPU budget so kreuzberg and the pipeline + # semaphore stop competing for cores. + max_concurrent = cpu_quota() + if mode is ExtractMode.PAGINATED: + return ExtractionConfig( + chunking=chunking, + pages=PageConfig(extract_pages=True, insert_page_markers=False), + ocr=ocr, + max_concurrent_extractions=max_concurrent, ) - return [(p.page, p.text) for p in pages] - - return await _vision_ocr_cached( - path, - source_name, - content_type, - ocr_fn=_ocr, - on_progress=on_progress, - page_texts_out=page_texts_out, - ) - - -async def _vision_image_ocr( - path: Path, - source_name: str, - content_type: str, - *, - on_progress: DetailedProgressCallback, - page_texts_out: list[PageTextRecord] | None = None, -) -> list[ChunkRecord]: - """Vision OCR an image through the worker pool, one page per frame. - - A multi-frame TIFF/GIF yields one OCR page per frame instead of silently - dropping every frame after the first. - """ - - async def _ocr() -> list[tuple[int, str]]: - page_pngs = await asyncio.to_thread(_image_page_pngs, path) - pages: list[tuple[int, str]] = [] - for page_num, png in enumerate(page_pngs, start=1): - text = await asyncio.to_thread(_ocr_image_png, png) - pages.append((page_num, text)) - return pages - - return await _vision_ocr_cached( - path, - source_name, - content_type, - ocr_fn=_ocr, - on_progress=on_progress, - page_texts_out=page_texts_out, - ) - - -def _ocr_image_png(png: bytes) -> str: - """OCR one rendered image page through the vision server.""" - return get_services().provider.vision_ocr( - png, cfg.vision_model, timeout=_effective_ocr_timeout() + return ExtractionConfig( + chunking=chunking, + output_format=MARKDOWN_OUTPUT, + ocr=ocr, + max_concurrent_extractions=max_concurrent, ) -def _image_page_pngs(path: Path) -> list[bytes]: - """Each frame of a (possibly multi-frame) image, re-encoded as PNG. - - A single-frame image yields one entry; a multipage TIFF/GIF yields one per - frame so every page reaches the projector instead of just the first. - """ - pages: list[bytes] = [] - with Image.open(path) as img: - for frame in ImageSequence.Iterator(img): - buf = BytesIO() - frame.convert("RGB").save(buf, format="PNG") - pages.append(buf.getvalue()) - return pages - - -def _run_tesseract_sync(path: Path) -> Any: - """Run kreuzberg Tesseract OCR with the worker's stderr redirected to /dev/null. - - Tesseract writes "Line cannot be recognized!!", "Image too small to - scale!!", and "Detected N diacritics" directly to fd 2 from inside libc. - Without the redirect those lines flood the TUI log file (1 000+ entries - per scanned PDF) and can bleed into the TUI itself. We hold the - suppression for just the duration of the extraction call so other - threads' stderr writes still go through. - """ - from kreuzberg import extract_file_sync - - from lilbee.core.system import stderr_suppressed - - with stderr_suppressed(): - return extract_file_sync(str(path), config=extraction_config(ExtractMode.PAGINATED_OCR)) - - -async def _tesseract_ocr_fallback( - path: Path, - source_name: str, - content_type: str, - *, - on_progress: DetailedProgressCallback, - page_texts_out: list[PageTextRecord] | None = None, -) -> list[ChunkRecord]: - """Tesseract OCR via ``asyncio.to_thread`` (no model load = no pool). - - ``cfg.tesseract_timeout`` caps the whole-document extract; 0 means - unlimited. Failures (including timeout) log a warning and return an - empty list so the caller can skip the file. OCR output is cached by file - content so a downstream failure doesn't force a re-OCR on retry. - """ - key = ocr_cache_key(file_hash(path), backend=TESSERACT_BACKEND, model=TESSERACT_BACKEND) - page_texts = load_ocr_pages(key) - if page_texts is None: - coro = asyncio.to_thread(_run_tesseract_sync, path) - try: - if cfg.tesseract_timeout > 0: - result = await asyncio.wait_for(coro, timeout=cfg.tesseract_timeout) - else: - result = await coro - except TimeoutError: - log.warning( - "Tesseract OCR exceeded %.0fs timeout on %s; skipping.", - cfg.tesseract_timeout, - source_name, - ) - return [] - except Exception: - log.warning("OCR via tesseract backend failed for %s.", source_name, exc_info=True) - return [] - - by_page: dict[int, list[str]] = {} - for chunk in result.chunks or []: - page = int(chunk.metadata.get("first_page") or 1) - by_page.setdefault(page, []).append(chunk.content) - page_texts = [(page, "\n".join(by_page[page])) for page in sorted(by_page)] - store_ocr_pages(key, page_texts) - _record_page_texts(page_texts, source_name, content_type, page_texts_out) - return await chunk_and_embed_pages(page_texts, source_name, content_type, on_progress) - - def _chunk_pages(page_texts: Sequence[tuple[int, str]]) -> list[tuple[int, str]]: - """Chunk each OCR page's text. Semantic chunking is off: a single OCR page - rarely spans multiple topics, so the semantic round-trip is not worth it.""" + """Chunk each page's text. Semantic chunking is off: a single page rarely spans + multiple topics, so the semantic round-trip is not worth it.""" return [ (page_num, chunk) for page_num, text in page_texts @@ -381,11 +157,11 @@ async def chunk_and_embed_pages( content_type: str, on_progress: DetailedProgressCallback, ) -> list[ChunkRecord]: - """Chunk per-page text and embed every chunk. Shared by OCR ingest and import.""" + """Chunk per-page text and embed every chunk. Used by the dataset import path.""" if not page_texts: return [] - # chunk_text runs kreuzberg's synchronous extractor; offload it so a long OCR + # chunk_text runs kreuzberg's synchronous extractor; offload it so a long # document does not stall sibling files sharing this event loop. all_chunks = await asyncio.to_thread(_chunk_pages, page_texts) if not all_chunks: @@ -417,16 +193,16 @@ def _capture_result_page_texts( content_type: str, page_texts_out: list[PageTextRecord] | None, ) -> None: - """Append a normal extraction's page texts to the export accumulator. + """Append an extraction's page texts to the export accumulator. - Paginated PDFs yield one row per ``result.pages`` entry; other documents - have no page split, so the full ``result.content`` is recorded as page 0. + Paginated documents yield one row per ``result.pages`` entry; others have no + page split, so the full ``result.content`` is recorded as page 0. """ if page_texts_out is None: return if result.pages: page_texts_out.extend( - _page_text_record(source_name, page["page_number"], page["content"], content_type) + _page_text_record(source_name, page.page_number, page.content, content_type) for page in result.pages ) elif result.content.strip(): @@ -434,7 +210,7 @@ def _capture_result_page_texts( def _warn_empty_ocr(source_name: str, media: str) -> None: - """Warn that OCR yielded no text and point to the vision-model remedy.""" + """Warn that extraction yielded no text and point to the vision-model remedy.""" log.warning( "Skipped %s: text extraction produced no usable text. " "For better results on %s, configure a vision model " @@ -444,92 +220,6 @@ def _warn_empty_ocr(source_name: str, media: str) -> None: ) -async def _handle_scanned_pdf_fallback( - path: Path, - source_name: str, - content_type: str, - result: ExtractionResult, - *, - quiet: bool, - on_progress: DetailedProgressCallback, - page_texts_out: list[PageTextRecord] | None = None, -) -> list[ChunkRecord]: - """Route a scanned PDF through the configured OCR backend. - - Vision OCR (pool-routed) runs when ``_should_run_ocr()`` is True - and a vision model is configured; otherwise the file falls back to - inline Tesseract. Both paths return chunk records; an empty list - means OCR found no usable text and the caller should skip the - file. - """ - del result # Both backends re-extract; the kreuzberg result is not reused. - if _effective_enable_ocr() is False: - # OCR explicitly disabled: skip entirely (vision and Tesseract) rather - # than paying the full Tesseract cost the config says is turned off. - log.info("OCR disabled; skipping scanned-PDF OCR for %s", source_name) - return [] - use_ocr = _should_run_ocr() - if use_ocr and cfg.vision_model: - log.info( - "Scanned PDF: using vision OCR for %s (model=%s)", - source_name, - cfg.vision_model, - ) - return await _vision_ocr_fallback( - path, - source_name, - content_type, - on_progress=on_progress, - quiet=quiet, - page_texts_out=page_texts_out, - ) - - log.info("Scanned PDF: falling back to Tesseract OCR for %s", source_name) - chunks = await _tesseract_ocr_fallback( - path, - source_name, - content_type, - on_progress=on_progress, - page_texts_out=page_texts_out, - ) - if not chunks: - _warn_empty_ocr(source_name, "scanned PDFs") - return chunks - - -async def _handle_image( - path: Path, - source_name: str, - content_type: str, - *, - on_progress: DetailedProgressCallback, - page_texts_out: list[PageTextRecord] | None = None, -) -> list[ChunkRecord]: - """OCR an image: vision OCR when a vision model is configured, else Tesseract. - - An image has no text layer to extract first, so it routes straight to OCR -- - the same downstream call a PDF page hits after it is rasterized to an image. - """ - if _effective_enable_ocr() is False: - # OCR explicitly disabled: an image has no text layer, so skip it rather - # than paying the full Tesseract cost the config says is turned off. - log.info("OCR disabled; skipping image OCR for %s", source_name) - return [] - if _should_run_ocr() and cfg.vision_model: - log.info("Image: using vision OCR for %s (model=%s)", source_name, cfg.vision_model) - return await _vision_image_ocr( - path, source_name, content_type, on_progress=on_progress, page_texts_out=page_texts_out - ) - - log.info("Image: falling back to Tesseract OCR for %s", source_name) - chunks = await _tesseract_ocr_fallback( - path, source_name, content_type, on_progress=on_progress, page_texts_out=page_texts_out - ) - if not chunks: - _warn_empty_ocr(source_name, "images") - return chunks - - async def ingest_document( path: Path, source_name: str, @@ -539,45 +229,42 @@ async def ingest_document( on_progress: DetailedProgressCallback = noop_callback, page_texts_out: list[PageTextRecord] | None = None, ) -> list[ChunkRecord]: - """Extract and chunk a document, embed, return records. + """Extract, chunk, and embed a document in a single kreuzberg pass. - Vision OCR is controlled by ``cfg.enable_ocr`` (see ``_should_run_ocr``). - When ``page_texts_out`` is given, per-page text is appended for export. + kreuzberg extracts native text and, where a page has none, OCRs it through the + registered backend (lilbee's vision model, or tesseract). Per-page OCR progress + is streamed as a running count via ``ocr_request``. ``quiet`` is accepted for + pipeline call compatibility. """ - # An image carries no text layer; route it straight to OCR (vision or Tesseract) - # instead of a no-op kreuzberg markdown extract that yields nothing for a scan. - if content_type == IMAGE_CONTENT_TYPE: - return await _handle_image( - path, source_name, content_type, on_progress=on_progress, page_texts_out=page_texts_out - ) + del quiet + from kreuzberg import extract_file - from kreuzberg import extract_file_sync + page_seen = 0 - config = extraction_config(content_type_to_mode(content_type)) - result = await asyncio.to_thread(extract_file_sync, str(path), config=config) - - if content_type == PDF_CONTENT_TYPE and not _has_meaningful_text(result): - return await _handle_scanned_pdf_fallback( - path, - source_name, - content_type, - result, - quiet=quiet, - on_progress=on_progress, - page_texts_out=page_texts_out, + def _tick() -> None: + nonlocal page_seen + page_seen += 1 + on_progress( + EventType.EXTRACT, + ExtractEvent(file=source_name, page=page_seen, total_pages=0), ) + with ocr_request(on_page=_tick, timeout=_effective_ocr_timeout()) as token: + config = extraction_config(content_type_to_mode(content_type), ocr_token=token) + # Async keeps the OCR page loop off this event loop's thread. + result = await extract_file(str(path), config=config) + if not result.chunks: + if content_type in (PDF_CONTENT_TYPE, IMAGE_CONTENT_TYPE): + _warn_empty_ocr(source_name, "scanned documents") return [] _capture_result_page_texts(result, source_name, content_type, page_texts_out) - # Fire one EXTRACT event per file so subscribers (chat /add, /sync, - # CLI Rich progress) can show "extracted N pages" before the embed - # phase starts; otherwise a 44MB PDF sits at file-level 0% for - # minutes. get_page_count is the canonical PDF page count; for - # non-paginated formats we fall back to the chunk count. - page_count = result.get_page_count() or len(result.chunks) + # One EXTRACT event per file so subscribers (chat /add, /sync, CLI Rich + # progress) show "extracted N pages" before the embed phase; result.pages is + # the canonical page list, falling back to the chunk count for non-paginated docs. + page_count = len(result.pages or []) or len(result.chunks or []) on_progress( EventType.EXTRACT, ExtractEvent(file=source_name, page=page_count, total_pages=page_count), @@ -587,21 +274,20 @@ async def ingest_document( vectors = await asyncio.to_thread( get_services().embedder.embed_batch, texts, source=source_name, on_progress=on_progress ) - return [ ChunkRecord( source=source_name, content_type=content_type, chunk_type=ChunkType.RAW, - page_start=chunk.metadata.get("first_page") or 0, - page_end=chunk.metadata.get("last_page") or 0, + page_start=chunk.metadata.first_page or 0, + page_end=chunk.metadata.last_page or 0, line_start=0, line_end=0, chunk=text, - chunk_index=chunk.metadata.get("chunk_index", idx), + chunk_index=chunk.metadata.chunk_index, vector=vec, ) - for idx, (chunk, text, vec) in enumerate(zip(result.chunks, texts, vectors, strict=True)) + for chunk, text, vec in zip(result.chunks, texts, vectors, strict=True) ] diff --git a/src/lilbee/data/ingest/ocr_cache.py b/src/lilbee/data/ingest/ocr_cache.py deleted file mode 100644 index af1e6ce38..000000000 --- a/src/lilbee/data/ingest/ocr_cache.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Content-addressed cache for per-page OCR text. - -OCR is the most expensive ingest step (a 595-page scanned manual runs ~18 min). -Its per-page output is cached keyed on the file's content hash plus the OCR -backend and model, so a failure in a downstream step (chunk, embed, or the -LanceDB write) no longer discards the whole OCR pass: the retry reads the cached -pages and goes straight to chunk + embed. -""" - -from __future__ import annotations - -import hashlib -import json -import logging -from pathlib import Path - -from lilbee.core.config import cfg - -log = logging.getLogger(__name__) - -# A cached page is serialized as a [page_number, text] pair. -_PAGE_ENTRY_LEN = 2 - -_CACHE_DIRNAME = "ocr_cache" -# Bump when the entry format or what counts as "the same OCR" changes, so old -# entries miss instead of being misread. -_CACHE_VERSION = "1" -_KEY_SEP = "\0" - - -def _cache_dir() -> Path: - return cfg.data_dir / _CACHE_DIRNAME - - -def ocr_cache_key(file_hash: str, *, backend: str, model: str, extra: str = "") -> str: - """Stable cache key for one file's OCR output under a backend/model/config tuple. - - ``file_hash`` ties the entry to exact file content (an edit changes the hash - and so misses); ``backend`` / ``model`` / ``extra`` capture what would change - the OCR text for the same bytes. - """ - payload = _KEY_SEP.join((_CACHE_VERSION, file_hash, backend, model, extra)) - return hashlib.sha256(payload.encode("utf-8")).hexdigest() - - -def load_ocr_pages(key: str) -> list[tuple[int, str]] | None: - """Return cached ``(page, text)`` pairs for *key*, or ``None`` on miss. - - A missing, unreadable, or malformed entry returns ``None`` so the caller - re-runs OCR rather than trusting partial data. - """ - path = _cache_dir() / f"{key}.json" - if not path.exists(): - return None - try: - raw = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - log.debug("OCR cache read failed for %s", key, exc_info=True) - return None - if not isinstance(raw, list): - return None - pages: list[tuple[int, str]] = [] - for entry in raw: - if ( - isinstance(entry, list) - and len(entry) == _PAGE_ENTRY_LEN - and isinstance(entry[0], int) - and isinstance(entry[1], str) - ): - pages.append((entry[0], entry[1])) - return pages - - -def store_ocr_pages(key: str, pages: list[tuple[int, str]]) -> None: - """Persist ``(page, text)`` pairs for *key*. - - Best-effort: a write failure is logged and swallowed (a cache miss next time - is harmless). Empty results are not cached so a failed OCR run is retried. - """ - if not pages: - return - cache_dir = _cache_dir() - try: - cache_dir.mkdir(parents=True, exist_ok=True) - path = cache_dir / f"{key}.json" - tmp = path.with_suffix(".json.tmp") - tmp.write_text(json.dumps([[page, text] for page, text in pages]), encoding="utf-8") - tmp.replace(path) - except OSError: - log.debug("OCR cache write failed for %s", key, exc_info=True) diff --git a/src/lilbee/data/ingest/types.py b/src/lilbee/data/ingest/types.py index 79bddf7ec..2e3ef5ae6 100644 --- a/src/lilbee/data/ingest/types.py +++ b/src/lilbee/data/ingest/types.py @@ -39,24 +39,24 @@ class FileChangePlan(NamedTuple): stat_backfills: list[SourceStatBackfill] -# Minimum total chars for extracted text to be considered meaningful. -# 50 chars ≈ 12 words: if a PDF yields less, it's almost certainly a scanned -# document with no embedded text layer. Text PDFs with even just a title page -# easily exceed this threshold; blank/scan-only PDFs yield 0 chars. -MIN_MEANINGFUL_CHARS = 50 - PDF_CONTENT_TYPE = "pdf" IMAGE_CONTENT_TYPE = "image" MARKDOWN_OUTPUT = "markdown" -TESSERACT_BACKEND = "tesseract" +MARKDOWN_MIME = "text/markdown" + + +class OcrBackendName(StrEnum): + """OCR backends lilbee selects in OcrConfig: kreuzberg's tesseract or lilbee's vision plugin.""" + + TESSERACT = "tesseract" + LILBEE_VISION = "lilbee-vision" class ExtractMode(StrEnum): - """Extraction topology: pagination / OCR / output format.""" + """Extraction topology: paginated (PDFs/images) vs markdown output (text formats).""" MARKDOWN = "markdown" PAGINATED = "paginated" - PAGINATED_OCR = "paginated_ocr" class ChunkRecord(TypedDict): diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py new file mode 100644 index 000000000..14ec40a2d --- /dev/null +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -0,0 +1,177 @@ +"""lilbee's vision model exposed as a kreuzberg custom OCR backend.""" + +from __future__ import annotations + +import json +import threading +import uuid +from contextlib import contextmanager +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version +from typing import TYPE_CHECKING, Any, Protocol, cast + +from lilbee.data.ingest.types import MARKDOWN_MIME, OcrBackendName +from lilbee.vision import resolve_ocr_prompt + +if TYPE_CHECKING: + from collections.abc import Callable, Generator + +# Token key inside OcrConfig.backend_options JSON. kreuzberg does not propagate +# contextvars into process_image (kreuzberg-4w9), so per-request state travels as +# a token on the config and is resolved through the registry below. +_REQUEST_TOKEN_KEY = "req" # noqa: S105 # JSON key name, not a secret + + +@dataclass(frozen=True) +class OcrRequestContext: + """Per-extraction state the backend needs but kreuzberg won't carry for it.""" + + on_page: Callable[[], None] | None = None + timeout: float = 0.0 + + +class _OcrRequestRegistry: + """Token-keyed request contexts; lock-guarded (process_image runs on kreuzberg threads).""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._by_token: dict[str, OcrRequestContext] = {} + + def register(self, ctx: OcrRequestContext) -> str: + token = uuid.uuid4().hex + with self._lock: + self._by_token[token] = ctx + return token + + def get(self, token: str | None) -> OcrRequestContext | None: + if token is None: + return None + with self._lock: + return self._by_token.get(token) + + def unregister(self, token: str) -> None: + with self._lock: + self._by_token.pop(token, None) + + +ocr_requests = _OcrRequestRegistry() + + +@contextmanager +def ocr_request( + *, on_page: Callable[[], None] | None = None, timeout: float = 0.0 +) -> Generator[str, None, None]: + """Register a per-extraction context and yield its token for OcrConfig.backend_options.""" + token = ocr_requests.register(OcrRequestContext(on_page=on_page, timeout=timeout)) + try: + yield token + finally: + ocr_requests.unregister(token) + + +def backend_options_for(token: str) -> str: + """Serialize a request token into the OcrConfig.backend_options string.""" + return json.dumps({_REQUEST_TOKEN_KEY: token}) + + +def _config_to_dict(config: str | dict[str, Any]) -> dict[str, Any]: + """Normalize the OcrConfig kreuzberg hands process_image into a dict. + + kreuzberg serializes the OcrConfig to a JSON string for the callback (rc.35); + an already-parsed dict is accepted as-is for forward compatibility. + """ + if isinstance(config, str): + try: + raw: object = json.loads(config) + except (ValueError, TypeError): + return {} + else: + raw = config + return cast("dict[str, Any]", raw) if isinstance(raw, dict) else {} + + +class _OcrConfigView: + """Typed reader over the kreuzberg OcrConfig (normalized to a dict).""" + + def __init__(self, config: dict[str, Any]) -> None: + self._config = config + + @property + def vlm_prompt(self) -> str | None: + return self._config.get("vlm_prompt") + + @property + def request_token(self) -> str | None: + raw = self._config.get("backend_options") + if isinstance(raw, str): + try: + raw = json.loads(raw) + except (ValueError, TypeError): + return None + if not isinstance(raw, dict): + return None + token = raw.get(_REQUEST_TOKEN_KEY) + return token if isinstance(token, str) else None + + +class _OcrFn(Protocol): + # Positional-only so the provider's vision_ocr (named png_bytes) matches structurally. + def __call__( + self, image_bytes: bytes, model: str, prompt: str, /, *, timeout: float + ) -> str: ... + + +def _lilbee_version() -> str: + try: + return version("lilbee") + except PackageNotFoundError: + return "0" + + +class VisionOcrBackend: + """Routes kreuzberg OCR calls to lilbee's vision model through the injected + ``ocr_fn`` (single-image OCR) and ``model_ref_fn`` (the active vision model).""" + + def __init__(self, *, ocr_fn: _OcrFn, model_ref_fn: Callable[[], str]) -> None: + self._ocr_fn = ocr_fn + self._model_ref_fn = model_ref_fn + + def name(self) -> str: + return OcrBackendName.LILBEE_VISION + + def version(self) -> str: + return _lilbee_version() + + def supported_languages(self) -> list[str]: + return [] + + def supports_language(self, lang: str) -> bool: + return True + + def initialize(self) -> None: ... + + def shutdown(self) -> None: ... + + def backend_type(self) -> str: + return "custom" + + def process_image(self, image_bytes: bytes, config: str | dict[str, Any]) -> dict[str, Any]: + # kreuzberg serializes the OcrConfig to a JSON string for the callback; + # normalize before reading fields. + view = _OcrConfigView(_config_to_dict(config)) + model = self._model_ref_fn() + prompt = view.vlm_prompt or resolve_ocr_prompt(model) + ctx = ocr_requests.get(view.request_token) + text = self._ocr_fn(image_bytes, model, prompt, timeout=ctx.timeout if ctx else 0.0) + if ctx is not None and ctx.on_page is not None: + ctx.on_page() + # kreuzberg deserializes this dict into its OCR result struct; all of these + # keys are required (kreuzberg-1mc). + return { + "content": text, + "mime_type": MARKDOWN_MIME, + "metadata": {}, + "tables": [], + "chunks": [], + "images": [], + } diff --git a/src/lilbee/providers/base.py b/src/lilbee/providers/base.py index 9d1240444..916406645 100644 --- a/src/lilbee/providers/base.py +++ b/src/lilbee/providers/base.py @@ -11,9 +11,8 @@ from pydantic import BaseModel if TYPE_CHECKING: - from lilbee.providers.roles import OcrBackend, WorkerRole + from lilbee.providers.roles import WorkerRole from lilbee.providers.warm_progress import WarmProgress - from lilbee.vision import PageText T_co = TypeVar("T_co", covariant=True) @@ -314,23 +313,6 @@ def vision_ocr( """OCR one page image; ``timeout`` seconds, ``None``/``0`` = no cap.""" ... - def pdf_ocr( - self, - path: Path, - *, - backend: OcrBackend, - model: str = "", - per_page_timeout_s: float | None = None, - quiet: bool = True, - on_progress: Callable[..., None] | None = None, - ) -> list[PageText]: - """OCR every page of a PDF, returning per-page text in input order. - - Backends that cannot OCR scanned PDFs locally raise - :class:`NotImplementedError`; ingest callers catch and log this. - """ - ... - def list_models(self) -> list[str]: """List available model identifiers.""" ... diff --git a/src/lilbee/providers/fleet/provider.py b/src/lilbee/providers/fleet/provider.py index 009ed134f..e8e1e8b59 100644 --- a/src/lilbee/providers/fleet/provider.py +++ b/src/lilbee/providers/fleet/provider.py @@ -14,8 +14,7 @@ import logging import re import threading -import time -from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from concurrent.futures import ThreadPoolExecutor from concurrent.futures import TimeoutError as FutureTimeoutError from contextlib import contextmanager from pathlib import Path @@ -43,8 +42,6 @@ ChatStreamItem, ChatToolResult, ClosableIterator, - OcrBackend, - PageText, ) from lilbee.providers.fleet.launch import InstanceLaunch @@ -183,8 +180,8 @@ def _supports_tools_cached(path_str: str, _mtime_ns: int) -> bool: class _VisionRequestGate: """Process-wide cap on concurrent vision-server requests at the fleet's OCR slots. - The ingest file fan-out runs many files at once and each file's ``pdf_ocr`` opens - its own ``vision_ocr_concurrency`` page pool, so without a shared cap the aggregate + The ingest file fan-out runs many files at once and kreuzberg OCRs their pages + through per-image ``vision_ocr`` calls, so without a shared cap the aggregate over-subscribes a single-replica vision server into a 429 storm. The semaphore is rebuilt to the configured capacity (``vision_replicas * vision_ocr_concurrency``) only while the gate is idle, so a capacity change never doubles the live cap. @@ -277,62 +274,6 @@ def _bounded_vision_chat( pool.shutdown(wait=False, cancel_futures=True) -def _ocr_pdf_page( - idx: int, - png: bytes, - *, - clients: list[LlamaServerClient], - ocr_prompt: str, - deadline: float | None, - page_path: Path, -) -> tuple[int, str]: - """OCR one rasterized page through the gated vision server; empty text on failure. - - Acquires the gate before reading the clock so queue time isn't billed against the - page's share of the document-wide deadline; an exhausted budget skips the page - rather than running it un-timed. - """ - from lilbee.vision import build_vision_messages - - messages = build_vision_messages(ocr_prompt, png) - with _VISION_GATE.slot(): - remaining = max(0.0, deadline - time.monotonic()) if deadline is not None else None - if remaining == 0.0: - log.warning( - "Vision OCR budget exhausted before page %d of %s; skipping.", - idx + 1, - page_path.name, - ) - return idx, "" - try: - return idx, _call_with_failover( - clients, lambda client: _vision_call(client, messages, remaining) - ) - except ProviderError: - # One failed/timed-out page yields empty text; siblings continue. - log.warning( - "Vision OCR failed for page %d of %s; skipping that page.", - idx + 1, - page_path.name, - exc_info=True, - ) - return idx, "" - - -def _pdf_drain_budget(total_pages: int, per_page_timeout_s: float | None) -> float | None: - """Total OCR wall-clock budget = pages*per_page + load grace, or None for no cap. - - Mirrors the in-process drain budget: one document-wide deadline rather than a - per-page cap, so a slow page borrows from fast ones and the vision model's cold - first-inference is absorbed by the grace instead of tripping a fixed page limit. - """ - from lilbee.core.config import cfg - - if not per_page_timeout_s or per_page_timeout_s <= 0: - return None - return total_pages * per_page_timeout_s + cfg.vision_load_budget_s - - class FleetProvider: """Routes every role to the managed llama-server fleet (a fleet-of-one on one box).""" @@ -700,87 +641,9 @@ def vision_ocr( clients, lambda client: _vision_call(client, messages, timeout) ) - def pdf_ocr( - self, - path: Path, - *, - backend: OcrBackend, - model: str = "", - per_page_timeout_s: float | None = None, - quiet: bool = True, - on_progress: Callable[..., None] | None = None, - ) -> list[PageText]: - """OCR each rasterized PDF page through the vision server. - - ``backend`` is ``Literal["vision"]`` (tesseract is run inline by the - ingest caller, never here). ``per_page_timeout_s`` caps each page's - request; ``quiet`` is accepted for protocol parity (the server emits no - Rich progress to suppress). Pages are numbered 1-based to match - ``PageText`` / ``ExtractEvent`` everywhere else in lilbee. - """ - from lilbee.core.config import cfg - from lilbee.runtime.progress import EventType, ExtractEvent - from lilbee.vision import ( - PageText, - pdf_page_count, - rasterize_pdf, - resolve_ocr_prompt, - ) - - del quiet # protocol parity; no server-side Rich progress to suppress. - self._require_configured_model(model, str(cfg.vision_model), WorkerRole.VISION) - clients = self._require_clients(WorkerRole.VISION) - # The model is fixed for the whole document, so resolve its prompt once. - ocr_prompt = resolve_ocr_prompt(model or str(cfg.vision_model)) - log.debug("OCR prompt for %s -> %r", model or cfg.vision_model, ocr_prompt) - total = pdf_page_count(path) - # One document-wide deadline (pages*per_page + load grace), not a per-page - # cap: each page gets whatever budget remains, so a slow page borrows from - # fast ones and the cold first-inference is covered, matching in-process OCR. - budget = _pdf_drain_budget(total, per_page_timeout_s) - deadline = (time.monotonic() + budget) if budget is not None else None - - _ocr = functools.partial( - _ocr_pdf_page, - clients=clients, - ocr_prompt=ocr_prompt, - deadline=deadline, - page_path=path, - ) - - # OCR pages concurrently (a single-page decode underuses the GPU; the vision - # server runs cfg.vision_ocr_concurrency batching slots). A bounded sliding - # window keeps that many pages in flight without rasterizing the whole PDF - # into memory; results are reassembled in page order. - concurrency = max(1, cfg.vision_ocr_concurrency) - raster = rasterize_pdf(path) - results: dict[int, str] = {} - with ThreadPoolExecutor(max_workers=concurrency) as pool: - pending: set[Future[tuple[int, str]]] = set() - - def _submit_next() -> bool: - page = next(raster, None) - if page is None: - return False - idx, png_bytes = page - pending.add(pool.submit(_ocr, idx, bytes(png_bytes))) - return True - - for _ in range(concurrency): - if not _submit_next(): - break - while pending: - completed, pending = wait(pending, return_when=FIRST_COMPLETED) - for done in completed: - page_idx, text = done.result() - results[page_idx] = text - if on_progress is not None: - on_progress( - EventType.EXTRACT, - ExtractEvent(file=path.name, page=page_idx + 1, total_pages=total), - ) - _submit_next() - return [PageText(idx + 1, results[idx]) for idx in sorted(results)] + # PDF/image OCR now runs inside kreuzberg via the registered lilbee-vision + # backend (see data.ingest.vision_ocr_backend); this provider only exposes + # single-image vision_ocr, which that backend calls. def rerank(self, query: str, candidates: list[str]) -> list[float]: clients = self._require_clients(WorkerRole.RERANK) diff --git a/src/lilbee/providers/fleet/swap_manager.py b/src/lilbee/providers/fleet/swap_manager.py index 354a0d1e7..3cede58ac 100644 --- a/src/lilbee/providers/fleet/swap_manager.py +++ b/src/lilbee/providers/fleet/swap_manager.py @@ -231,6 +231,7 @@ def _write_state(self) -> None: _STATE_KEY_NAME: _LLAMA_SWAP_PROCESS_NAME, _STATE_KEY_MEMBER_PORTS: self._member_ports, } + self._state_path.parent.mkdir(parents=True, exist_ok=True) tmp_path = self._state_path.with_name( f"{_STATE_TMP_PREFIX}{self._state_path.name}{_STATE_TMP_SUFFIX}" ) diff --git a/src/lilbee/providers/routing_provider.py b/src/lilbee/providers/routing_provider.py index 31e8923c4..0e4537524 100644 --- a/src/lilbee/providers/routing_provider.py +++ b/src/lilbee/providers/routing_provider.py @@ -21,9 +21,8 @@ ) from lilbee.providers.litellm_sdk import LitellmSdkBackend from lilbee.providers.model_ref import ProviderModelRef, parse_model_ref, routes_to_native_gguf -from lilbee.providers.roles import OcrBackend, WorkerRole +from lilbee.providers.roles import WorkerRole from lilbee.providers.sdk_llm_provider import SdkLLMProvider -from lilbee.vision import PageText if TYPE_CHECKING: from lilbee.providers.warm_progress import WarmProgress @@ -173,33 +172,6 @@ def vision_ocr( ref = parse_model_ref(model) return self._pick_backend(ref).vision_ocr(png_bytes, model, prompt, timeout=timeout) - def pdf_ocr( - self, - path: Path, - *, - backend: OcrBackend, - model: str = "", - per_page_timeout_s: float | None = None, - quiet: bool = True, - on_progress: Callable[..., None] | None = None, - ) -> list[PageText]: - """Dispatch by ``model``'s ref prefix, same rules as :meth:`vision_ocr`. - - Hosted refs reach :class:`SdkLLMProvider`, which raises - ``NotImplementedError`` for PDF OCR; native refs reach the - local llama-server engine. ``model`` is empty when the caller wants - the configured ``cfg.vision_model`` to drive the dispatch. - """ - ref = parse_model_ref(model or cfg.vision_model) - return self._pick_backend(ref).pdf_ocr( - path, - backend=backend, - model=model, - per_page_timeout_s=per_page_timeout_s, - quiet=quiet, - on_progress=on_progress, - ) - def list_models(self) -> list[str]: """Return the union of native and SDK-visible models. diff --git a/src/lilbee/providers/sdk_llm_provider.py b/src/lilbee/providers/sdk_llm_provider.py index a0877e741..3ad01f719 100644 --- a/src/lilbee/providers/sdk_llm_provider.py +++ b/src/lilbee/providers/sdk_llm_provider.py @@ -36,7 +36,6 @@ from lilbee.providers.local_servers import LOCAL_SERVER_KEYS from lilbee.providers.local_servers.config_urls import base_url_for, configured_local_servers from lilbee.providers.model_ref import ProviderModelRef, parse_model_ref, translate_options -from lilbee.providers.roles import OcrBackend from lilbee.providers.sdk_backend import ( PROVIDER_KEYS, CompletionRequest, @@ -44,7 +43,6 @@ LlmSdkBackend, RerankRequest, ) -from lilbee.vision import PageText log = logging.getLogger(__name__) @@ -301,23 +299,6 @@ def vision_ocr( ) return result.text - def pdf_ocr( - self, - path: Path, - *, - backend: OcrBackend, - model: str = "", - per_page_timeout_s: float | None = None, - quiet: bool = True, - on_progress: Callable[..., None] | None = None, - ) -> list[PageText]: - """SDK backend cannot rasterise PDFs locally; ingest callers fall back.""" - del path, backend, model, per_page_timeout_s, quiet, on_progress - raise NotImplementedError( - "Hosted models do not support scanned-PDF OCR. " - "Set LILBEE_VISION_MODEL to a local GGUF vision model to enable it." - ) - def list_models(self) -> list[str]: """List models across every configured local server. diff --git a/src/lilbee/runtime/progress/types.py b/src/lilbee/runtime/progress/types.py index 255b90143..86256e96f 100644 --- a/src/lilbee/runtime/progress/types.py +++ b/src/lilbee/runtime/progress/types.py @@ -83,10 +83,10 @@ class BatchProgressEvent(BaseModel): class ExtractEvent(BaseModel): """Emitted with page-level extraction progress. - Vision PDF OCR fires one event per page (``page < total_pages``); - plain (non-OCR) extraction fires once per file with - ``page == total_pages`` so subscribers see "extracted N pages" - before the embed phase ticks. + OCR fires one event per page as kreuzberg processes it, as a running count + with ``total_pages == 0`` (the total is unknown mid-extraction). Extraction + then fires once per file with ``page == total_pages`` so subscribers see + "extracted N pages" before the embed phase ticks. """ file: str diff --git a/src/lilbee/server/handlers/models.py b/src/lilbee/server/handlers/models.py index 4c63c55de..8cf4cf31f 100644 --- a/src/lilbee/server/handlers/models.py +++ b/src/lilbee/server/handlers/models.py @@ -280,6 +280,8 @@ async def set_embedding_model(model: str) -> SetModelResponse: async def set_vision_model(model: str) -> SetModelResponse: """Switch vision OCR model. Empty string unsets it (vision OCR disabled).""" normalized = _require_model_for_task(model, ModelTask.VISION, allow_empty=True) + # The OCR-backend (un)registration is handled by apply_settings_update's role + # reload fan-out inside _set_model, covering REST/MCP/TUI/CLI uniformly. return await _set_model("vision_model", normalized) diff --git a/src/lilbee/vision.py b/src/lilbee/vision.py index bc0e35a11..67079f2a3 100644 --- a/src/lilbee/vision.py +++ b/src/lilbee/vision.py @@ -1,35 +1,10 @@ -"""Helpers for PDF rasterisation and vision-model OCR. +"""Vision-model OCR helpers: prompt resolution and OpenAI-compatible image messages. -Multi-page vision OCR runs through ``FleetProvider.pdf_ocr``, which rasterises -each page and sends it to the vision server; this module hosts the small helpers -(page count, rasterisation, prompt + chat-message construction, and the shared -:class:`PageText` / :class:`PdfOcrChunk` types) that the provider and its callers -share. +PDF rasterisation and the page loop now live inside kreuzberg (the registered +lilbee-vision OCR backend); this module only builds the single-image request the +provider's ``vision_ocr`` sends to the vision server. """ -import logging -from collections.abc import Iterator -from pathlib import Path -from typing import NamedTuple - -log = logging.getLogger(__name__) - - -class PageText(NamedTuple): - """Extracted text for a single PDF page.""" - - page: int - text: str - - -class PdfOcrChunk(NamedTuple): - """One streaming PDF-OCR worker frame: page index, total pages, page text.""" - - page: int - total: int - text: str - - OCR_PROMPT = ( "Extract ALL text from this page as clean markdown. " "Preserve table structure using markdown table syntax. " @@ -53,25 +28,6 @@ def resolve_ocr_prompt(model_ref: str) -> str: return OCR_PROMPT -_RASTER_DPI = 150 - - -def pdf_page_count(path: Path) -> int: - """Return the number of pages in a PDF without rasterizing.""" - from kreuzberg import PdfPageIterator # lazy: heavy dependency - - it = PdfPageIterator(str(path), dpi=_RASTER_DPI) - return len(it) - - -def rasterize_pdf(path: Path) -> Iterator[tuple[int, bytes]]: - """Yield (0-based index, PNG bytes) for each page of a PDF.""" - from kreuzberg import PdfPageIterator # lazy: heavy dependency - - with PdfPageIterator(str(path), dpi=_RASTER_DPI) as pages: - yield from pages - - def _png_to_data_url(png_bytes: bytes) -> str: """Convert raw PNG bytes to a base64 data URL for OpenAI-compatible messages.""" import base64 diff --git a/tests/integration/test_pdf_integration.py b/tests/integration/test_pdf_integration.py index ce05e7a74..9d5905cfe 100644 --- a/tests/integration/test_pdf_integration.py +++ b/tests/integration/test_pdf_integration.py @@ -134,7 +134,7 @@ async def test_tesseract_extracts_text(self): """Tesseract OCR produces non-empty text from the scanned PDF fixture.""" from kreuzberg import ExtractionConfig, OcrConfig, extract_file - config = ExtractionConfig(ocr=OcrConfig()) + config = ExtractionConfig(ocr=OcrConfig(backend="tesseract"), force_ocr=True) result = await extract_file(str(SCANNED_PDF), config=config) assert len(result.content.strip()) > 0, "Tesseract produced empty text" @@ -146,7 +146,7 @@ async def test_tesseract_extracts_known_phrases(self): """Tesseract OCR captures key phrases from the scanned document.""" from kreuzberg import ExtractionConfig, OcrConfig, extract_file - config = ExtractionConfig(ocr=OcrConfig()) + config = ExtractionConfig(ocr=OcrConfig(backend="tesseract"), force_ocr=True) result = await extract_file(str(SCANNED_PDF), config=config) text_lower = result.content.lower() # At least some of the rendered text should be recognized @@ -171,41 +171,38 @@ def _vision_model_available() -> bool: class TestVisionOcrFallback: - """Verify vision model OCR fallback on scanned PDFs.""" + """Vision OCR through the registered lilbee-vision kreuzberg backend.""" + + async def _vision_extract(self) -> str: + from kreuzberg import ExtractionConfig, OcrConfig, extract_file + + from lilbee.app.services import get_services, sync_vision_ocr_backend + from lilbee.data.ingest.types import OcrBackendName + + sync_vision_ocr_backend(get_services().provider) + config = ExtractionConfig( + ocr=OcrConfig(backend=OcrBackendName.LILBEE_VISION), force_ocr=True + ) + result = await extract_file(str(SCANNED_PDF), config=config) + return result.content @pytest.mark.skipif( not _vision_model_available(), - reason="No vision-capable chat model available locally", + reason="No vision-capable model available locally", ) async def test_vision_extracts_text(self): - """Vision model OCR produces non-empty text from the scanned PDF fixture.""" - from lilbee.app.services import get_services - - page_texts = get_services().provider.pdf_ocr( - SCANNED_PDF, - backend="vision", - model=cfg.chat_model, - per_page_timeout_s=cfg.ocr_timeout, - ) - all_text = " ".join(text for _, text in page_texts) - assert len(all_text.strip()) > 0, "Vision OCR produced empty text" + """Vision OCR via the registered backend produces non-empty text.""" + content = await self._vision_extract() + assert len(content.strip()) > 0, "Vision OCR produced empty text" @pytest.mark.skipif( not _vision_model_available(), - reason="No vision-capable chat model available locally", + reason="No vision-capable model available locally", ) async def test_vision_extracts_known_phrases(self): - """Vision model OCR captures key phrases from the scanned document.""" - from lilbee.app.services import get_services - - page_texts = get_services().provider.pdf_ocr( - SCANNED_PDF, - backend="vision", - model=cfg.chat_model, - per_page_timeout_s=cfg.ocr_timeout, - ) - all_text = " ".join(text for _, text in page_texts).lower() + """Vision OCR via the registered backend captures key phrases.""" + text_lower = (await self._vision_extract()).lower() recognized = any( - phrase in all_text for phrase in ["oil", "maintenance", "filter", "quarts", "engine"] + phrase in text_lower for phrase in ["oil", "maintenance", "filter", "quarts", "engine"] ) - assert recognized, f"No expected phrases found in vision output: {all_text[:200]}" + assert recognized, f"No expected phrases found in vision output: {text_lower[:200]}" diff --git a/tests/server/test_handlers.py b/tests/server/test_handlers.py index ae53ad70a..d410863b6 100644 --- a/tests/server/test_handlers.py +++ b/tests/server/test_handlers.py @@ -82,11 +82,12 @@ def _make_kreuzberg_result(text: str = "Some extracted text. " * 20, num_chunks: chunk_text = text[i * len(text) // num_chunks : (i + 1) * len(text) // num_chunks] chunk = mock.MagicMock() chunk.content = chunk_text - chunk.metadata = {"chunk_index": i} + chunk.metadata = mock.MagicMock(chunk_index=i, first_page=None, last_page=None) chunks.append(chunk) result = mock.MagicMock() result.chunks = chunks result.content = text + result.pages = [] return result diff --git a/tests/test_chunker.py b/tests/test_chunker.py index 4b6724521..e38bcabf6 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -118,14 +118,16 @@ def test_semantic_enabled_uses_semantic_chunker_with_embedding(self, monkeypatch assert result.embedding is not None def test_semantic_respects_max_chars_when_embedding_present(self, monkeypatch): - """With an embedding attached kreuzberg honors max_chars on the semantic path.""" + """With an embedding attached kreuzberg honors max_characters on the semantic path.""" from lilbee.core.config import cfg from lilbee.data.chunk import CHARS_PER_TOKEN, build_chunking_config monkeypatch.setattr(cfg, "semantic_chunking", True) monkeypatch.setattr(cfg, "chunk_size", 512) result = build_chunking_config() - assert result.max_chars == 512 * CHARS_PER_TOKEN + assert result.max_characters == 512 * CHARS_PER_TOKEN + assert result.embedding is not None + assert result.embedding.model == "fast" def test_char_budget_when_disabled(self, monkeypatch): from lilbee.core.config import cfg @@ -136,8 +138,8 @@ def test_char_budget_when_disabled(self, monkeypatch): monkeypatch.setattr(cfg, "chunk_overlap", 100) result = build_chunking_config() assert result.chunker_type == "text" - assert result.max_chars == 512 * CHARS_PER_TOKEN - assert result.max_overlap == 100 * CHARS_PER_TOKEN + assert result.max_characters == 512 * CHARS_PER_TOKEN + assert result.overlap == 100 * CHARS_PER_TOKEN assert result.embedding is None def test_disabled_does_not_attach_embedding(self, monkeypatch): diff --git a/tests/test_cli_model.py b/tests/test_cli_model.py index 6cbbe451e..d6cd3ad66 100644 --- a/tests/test_cli_model.py +++ b/tests/test_cli_model.py @@ -483,6 +483,33 @@ def test_already_installed_short_circuits(self, fake_manager, native_manifests): assert result.status == PullStatus.ALREADY_INSTALLED assert fake_manager.pull_calls == [] + def test_already_installed_vision_ensures_mmproj(self): + """A cached vision model with a missing mmproj gets its projector fetched (bb-7yd).""" + from types import SimpleNamespace + + from lilbee.catalog.types import ModelTask + + entry = SimpleNamespace(task=ModelTask.VISION, hf_repo="org/vis-GGUF") + with ( + patch("lilbee.catalog.resolve_pull_target", return_value=entry), + patch("lilbee.catalog.download_mmproj") as dl, + ): + model_mod._ensure_vision_projector("org/vis-GGUF/model.gguf") + dl.assert_called_once_with(entry) + + def test_ensure_vision_projector_noop_for_non_vision(self): + from types import SimpleNamespace + + from lilbee.catalog.types import ModelTask + + entry = SimpleNamespace(task=ModelTask.CHAT, hf_repo="org/chat-GGUF") + with ( + patch("lilbee.catalog.resolve_pull_target", return_value=entry), + patch("lilbee.catalog.download_mmproj") as dl, + ): + model_mod._ensure_vision_projector("org/chat-GGUF/model.gguf") + dl.assert_not_called() + def test_pull_native_invokes_manager_and_callbacks(self, fake_manager, native_manifests): events = [] target = "Qwen/Qwen3-8B-GGUF/Qwen3-8B-Q4_K_M.gguf" diff --git a/tests/test_dynamic_settings_eviction.py b/tests/test_dynamic_settings_eviction.py index 01fa37597..e4d8828f9 100644 --- a/tests/test_dynamic_settings_eviction.py +++ b/tests/test_dynamic_settings_eviction.py @@ -36,6 +36,11 @@ def reload_role(self, role: object, *, wait: bool = False) -> None: def drop_loaded_models_async(self) -> None: self.dropped += 1 + def vision_ocr( + self, image_bytes: bytes, model: str, prompt: str = "", *, timeout: float | None = None + ) -> str: + return "" + @pytest.fixture(autouse=True) def _isolated_cfg(tmp_path): @@ -272,12 +277,18 @@ def test_reconcile_embedding_dim_leaves_dim_when_unreadable_or_matching(monkeypa assert cfg.embedding_dim == 768 # already matches -> untouched -def test_chat_and_vision_model_change_reloads_those_roles(): +def test_chat_and_vision_model_change_reloads_those_roles(monkeypatch): """A chat_model / vision_model swap reloads that role's server so the next request uses the new model. The fleet serves the configured model per role and rejects per-call overrides, so the reload is required (not optional).""" from lilbee.providers.roles import WorkerRole + # The vision-model reload also (un)registers the OCR backend; keep that global + # kreuzberg state out of this unit test. + monkeypatch.setattr("kreuzberg.list_ocr_backends", list) + monkeypatch.setattr("kreuzberg.register_ocr_backend", lambda backend: None) + monkeypatch.setattr("kreuzberg.unregister_ocr_backend", lambda name: None) + provider = _install_recording_provider() try: apply_settings_update({"chat_model": "Qwen/Qwen3-0.6B-GGUF"}) diff --git a/tests/test_fleet_provider.py b/tests/test_fleet_provider.py index d71c8ec31..9295b572d 100644 --- a/tests/test_fleet_provider.py +++ b/tests/test_fleet_provider.py @@ -493,25 +493,6 @@ def _hold() -> None: assert ran == 8 # every caller ran, just serialized through the gate -def test_ocr_pdf_page_skips_when_budget_exhausted(monkeypatch) -> None: - # A page that waited out the document deadline while queued is skipped (empty - # text), not run un-timed -- a 0 timeout would disable the per-page cap. - monkeypatch.setattr(prov_mod._VISION_GATE, "_semaphore", None) - monkeypatch.setattr("lilbee.vision.build_vision_messages", lambda *_a, **_k: []) - called: list[object] = [] - monkeypatch.setattr(prov_mod, "_vision_call", lambda *a, **_k: called.append(a) or "ocr") - idx, text = prov_mod._ocr_pdf_page( - 3, - b"\x89PNG", - clients=[_fake_client()], - ocr_prompt="describe", - deadline=time.monotonic() - 1.0, # already past - page_path=Path("doc.pdf"), - ) - assert (idx, text) == (3, "") - assert called == [] # _vision_call never ran - - def test_chat_streams_from_server() -> None: client = _fake_client(0) client.chat_stream_items.return_value = iter(["a", "b"]) @@ -670,116 +651,6 @@ def test_supports_tools_tolerates_unstattable_path(monkeypatch, _clear_tools_cac assert FleetProvider().supports_tools("org/repo/chat.gguf") is True -def test_pdf_ocr_ocrs_each_page_over_vision_server(monkeypatch) -> None: - from lilbee.runtime.progress import EventType - from lilbee.vision import PageText - - client = _fake_client(0) - client.chat.side_effect = ["page one", "page two"] - p = _provider_with_clients({WorkerRole.VISION: [client]}) - monkeypatch.setattr(cfg, "vision_model", "") # empty model arg -> configured - monkeypatch.setattr(cfg, "vision_ocr_concurrency", 1) # sequential: side_effect by call order - monkeypatch.setattr("lilbee.vision.pdf_page_count", lambda _p: 2) - monkeypatch.setattr( - "lilbee.vision.rasterize_pdf", lambda _p: iter([(0, b"png0"), (1, b"png1")]) - ) - events: list[tuple] = [] - result = p.pdf_ocr( - Path("doc.pdf"), - backend="vision", # type: ignore[arg-type] - on_progress=lambda etype, evt: events.append((etype, evt.page, evt.total_pages)), - ) - assert result == [PageText(1, "page one"), PageText(2, "page two")] - assert events == [(EventType.EXTRACT, 1, 2), (EventType.EXTRACT, 2, 2)] - - -def test_pdf_ocr_runs_pages_concurrently_and_preserves_order(monkeypatch) -> None: - # OCR fans pages across the vision server's batching slots; results must still - # come back in page order, and more than one page must be in flight at once. - import threading - import time as _time - - monkeypatch.setattr(cfg, "vision_model", "") - monkeypatch.setattr(cfg, "vision_ocr_concurrency", 4) - n = 8 - monkeypatch.setattr("lilbee.vision.pdf_page_count", lambda _p: n) - monkeypatch.setattr( - "lilbee.vision.rasterize_pdf", lambda _p: iter([(i, f"png{i}".encode()) for i in range(n)]) - ) - lock = threading.Lock() - inflight = {"now": 0, "max": 0} - - def _vision(_client, _messages, _timeout): - with lock: - inflight["now"] += 1 - inflight["max"] = max(inflight["max"], inflight["now"]) - _time.sleep(0.02) - with lock: - inflight["now"] -= 1 - return "ocr" - - monkeypatch.setattr(prov_mod, "_vision_call", _vision) - p = _provider_with_clients({WorkerRole.VISION: [_fake_client(0)]}) - result = p.pdf_ocr(Path("doc.pdf"), backend="vision") # type: ignore[arg-type] - assert [pt.page for pt in result] == list(range(1, n + 1)) # reassembled in order - assert inflight["max"] >= 2 # pages ran concurrently, not one at a time - - -def test_pdf_drain_budget_totals_pages_plus_load_grace(monkeypatch) -> None: - """Budget is one document-wide pool: pages*per_page + load grace, else uncapped.""" - monkeypatch.setattr(cfg, "vision_load_budget_s", 300.0) - assert prov_mod._pdf_drain_budget(2, 120.0) == 540.0 - assert prov_mod._pdf_drain_budget(5, None) is None - assert prov_mod._pdf_drain_budget(5, 0.0) is None - - -def test_pdf_ocr_spends_one_document_budget_across_pages(monkeypatch) -> None: - """Each page gets the remaining doc budget, not a fixed per-page cap.""" - from lilbee.vision import PageText - - p = _provider_with_clients({WorkerRole.VISION: [_fake_client(0)]}) - monkeypatch.setattr(cfg, "vision_model", "") - monkeypatch.setattr(cfg, "vision_load_budget_s", 300.0) - monkeypatch.setattr("lilbee.vision.pdf_page_count", lambda _p: 2) - monkeypatch.setattr( - "lilbee.vision.rasterize_pdf", lambda _p: iter([(0, b"png0"), (1, b"png1")]) - ) - seen: list[float | None] = [] - - def _capture(_client, _messages, timeout): - seen.append(timeout) - return "ocr" - - monkeypatch.setattr(prov_mod, "_vision_call", _capture) - result = p.pdf_ocr(Path("doc.pdf"), backend="vision", per_page_timeout_s=120.0) # type: ignore[arg-type] - assert result == [PageText(1, "ocr"), PageText(2, "ocr")] - # Budget is 2*120 + 300 = 540; pages run concurrently, so each draws nearly - # the full remaining budget (far above any 120 cap), in either capture order. - assert seen[0] == pytest.approx(540.0, abs=1.0) - assert seen[1] == pytest.approx(540.0, abs=1.0) - assert all(t is not None and t > 120.0 for t in seen) - - -def test_pdf_ocr_without_per_page_timeout_runs_uncapped(monkeypatch) -> None: - """No per-page cap means an uncapped (None) budget on every page.""" - p = _provider_with_clients({WorkerRole.VISION: [_fake_client(0)]}) - monkeypatch.setattr(cfg, "vision_model", "") - monkeypatch.setattr("lilbee.vision.pdf_page_count", lambda _p: 1) - monkeypatch.setattr("lilbee.vision.rasterize_pdf", lambda _p: iter([(0, b"png0")])) - seen: list[float | None] = [] - monkeypatch.setattr(prov_mod, "_vision_call", lambda *a: seen.append(a[2]) or "ocr") - p.pdf_ocr(Path("doc.pdf"), backend="vision", per_page_timeout_s=None) # type: ignore[arg-type] - assert seen == [None] - - -def test_pdf_ocr_without_server_raises() -> None: - from lilbee.providers.base import ProviderError - - p = _provider_with_clients({}) - with pytest.raises(ProviderError, match="No vision model server is running"): - p.pdf_ocr(Path("doc.pdf"), backend="vision") # type: ignore[arg-type] - - # --- llama-swap lifecycle ---------------------------------------------------- @@ -1499,27 +1370,6 @@ def test_vision_ocr_fails_over_to_healthy_replica(self, monkeypatch) -> None: assert dead.healthy is False assert alive.calls == 1 - def test_vision_pdf_page_fails_over_to_healthy_replica(self, monkeypatch) -> None: - """The per-page PDF OCR path fails a dead replica over too, so a page is - OCR'd by a live replica rather than skipped to empty text (bb-7jg1.5).""" - import httpx as _httpx - - from lilbee.providers.fleet import provider as prov - - dead = _FakeReplica(fail=_httpx.ConnectError("refused")) - alive = _FakeReplica(in_flight=5) - idx, text = prov._ocr_pdf_page( - 0, - b"\x89PNG", - clients=[dead, alive], - ocr_prompt="read it", - deadline=None, - page_path=Path("doc.pdf"), - ) - assert (idx, text) == (0, "ocr text") - assert dead.healthy is False - assert alive.calls == 1 - def test_successful_call_restores_an_unhealthy_replica(self) -> None: only = _FakeReplica() only.mark_unhealthy() @@ -1641,30 +1491,6 @@ def test_bounded_call_passes_the_deadline_to_the_http_request(self) -> None: assert prov_mod._vision_call(client, [{"role": "user", "content": "x"}], 9.0) == "text" assert client.chat.call_args.kwargs["timeout"] == 9.0 - def test_pdf_ocr_one_timed_out_page_does_not_abort_siblings(self, monkeypatch) -> None: - from lilbee.providers.base import ProviderError - from lilbee.vision import PageText - - monkeypatch.setattr(cfg, "vision_model", "") - monkeypatch.setattr(cfg, "vision_ocr_concurrency", 1) - monkeypatch.setattr("lilbee.vision.pdf_page_count", lambda _p: 2) - monkeypatch.setattr( - "lilbee.vision.rasterize_pdf", lambda _p: iter([(0, b"png0"), (1, b"png1")]) - ) - - failed_once: list[bool] = [] - - def _vision(_client, _messages, _timeout) -> str: - if not failed_once: - failed_once.append(True) - raise ProviderError("Vision OCR timed out after 1s.", provider="llama-server") - return "page two" - - monkeypatch.setattr(prov_mod, "_vision_call", _vision) - p = _provider_with_clients({WorkerRole.VISION: [_fake_client(0)]}) - result = p.pdf_ocr(Path("doc.pdf"), backend="vision") # type: ignore[arg-type] - assert result == [PageText(1, ""), PageText(2, "page two")] - class TestReloadSingleFlight: def test_concurrent_reload_calls_dispatch_one_reload(self, monkeypatch) -> None: diff --git a/tests/test_formats.py b/tests/test_formats.py index d0fcd97f5..c9eee1446 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -8,7 +8,6 @@ from __future__ import annotations from unittest import mock -from unittest.mock import Mock import pytest @@ -57,17 +56,20 @@ def _make_kreuzberg_result(text="Extracted content. " * 10, num_chunks=1): chunk_text = text[i * len(text) // num_chunks : (i + 1) * len(text) // num_chunks] chunk = mock.MagicMock() chunk.content = chunk_text - chunk.metadata = { - "byte_start": 0, - "byte_end": len(chunk_text), - "chunk_index": i, - "total_chunks": num_chunks, - "token_count": None, - } + chunk.metadata = mock.MagicMock( + byte_start=0, + byte_end=len(chunk_text), + chunk_index=i, + total_chunks=num_chunks, + token_count=None, + first_page=None, + last_page=None, + ) chunks.append(chunk) result = mock.MagicMock() result.chunks = chunks result.content = text + result.pages = [] return result @@ -77,8 +79,8 @@ def _make_kreuzberg_result(text="Extracted content. " * 10, num_chunks=1): @mock.patch( - "kreuzberg.extract_file_sync", - new_callable=Mock, + "kreuzberg.extract_file", + new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result(), ) class TestSyncDocx: @@ -91,8 +93,8 @@ async def test_docx_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "kreuzberg.extract_file_sync", - new_callable=Mock, + "kreuzberg.extract_file", + new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result(), ) class TestSyncXlsx: @@ -105,8 +107,8 @@ async def test_xlsx_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "kreuzberg.extract_file_sync", - new_callable=Mock, + "kreuzberg.extract_file", + new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result(), ) class TestSyncPptx: @@ -124,8 +126,8 @@ async def test_pptx_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "kreuzberg.extract_file_sync", - new_callable=Mock, + "kreuzberg.extract_file", + new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result(), ) class TestSyncEpub: @@ -143,8 +145,8 @@ async def test_epub_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "kreuzberg.extract_file_sync", - new_callable=Mock, + "kreuzberg.extract_file", + new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result(), ) class TestSyncImage: @@ -222,8 +224,8 @@ async def test_code_file_syncs(self, isolated_env, filename, fixture): @mock.patch( - "kreuzberg.extract_file_sync", - new_callable=Mock, + "kreuzberg.extract_file", + new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result(), ) class TestSyncCsvTsv: diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 783a52a62..3e66f1f75 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -4,7 +4,7 @@ import sys from pathlib import Path from unittest import mock -from unittest.mock import MagicMock, Mock +from unittest.mock import MagicMock import pytest @@ -142,19 +142,17 @@ def _make_kreuzberg_result( chunks = [] for i in range(num_chunks): chunk_text = text[i * len(text) // num_chunks : (i + 1) * len(text) // num_chunks] - metadata = { - "byte_start": 0, - "byte_end": len(chunk_text), - "chunk_index": i, - "total_chunks": num_chunks, - "token_count": None, - } - if has_pages: - metadata["first_page"] = i + 1 - metadata["last_page"] = i + 1 chunk = mock.MagicMock() chunk.content = chunk_text - chunk.metadata = metadata + chunk.metadata = mock.MagicMock( + byte_start=0, + byte_end=len(chunk_text), + chunk_index=i, + total_chunks=num_chunks, + token_count=None, + first_page=(i + 1) if has_pages else None, + last_page=(i + 1) if has_pages else None, + ) chunks.append(chunk) result = mock.MagicMock() @@ -162,7 +160,7 @@ def _make_kreuzberg_result( result.content = text result.document = document result.pages = ( - [{"page_number": i + 1, "content": chunks[i].content} for i in range(num_chunks)] + [mock.MagicMock(page_number=i + 1, content=chunks[i].content) for i in range(num_chunks)] if has_pages else [] ) @@ -178,7 +176,9 @@ def _make_empty_result(): return result -@mock.patch("kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result()) +@mock.patch( + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() +) class TestSync: async def test_empty_documents_dir(self, mock_extract_file, isolated_env): from lilbee.data.ingest import SyncResult, sync @@ -683,7 +683,9 @@ async def _fail(path, name, ct, **kwargs): assert "qflaky.txt" not in result.updated -@mock.patch("kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result()) +@mock.patch( + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() +) class TestSyncCancellation: """Tests for cancel support and atomic per-file delete in sync.""" @@ -825,7 +827,9 @@ async def test_cancel_preserves_old_chunks_for_modified_file( class TestIngestHelpers: """Cover edge cases in ingest_document and ingest_code_sync.""" - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_empty_result()) + @mock.patch( + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_empty_result() + ) async def testingest_document_empty_chunks(self, mock_extract_file, isolated_env): """Document that produces no chunks returns empty list.""" from lilbee.data.ingest import ingest_document @@ -879,7 +883,7 @@ class _FakeResult: assert "# File: pkg/mod.py" in joined assert str(f) not in joined # absolute path must never leak into content - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) + @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) async def testingest_document_pdf_with_pages(self, mock_kf, isolated_env): """PDF document returns records with page metadata.""" mock_kf.return_value = _make_kreuzberg_result( @@ -899,7 +903,7 @@ async def testingest_document_pdf_with_pages(self, mock_kf, isolated_env): class TestCancellation: @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_cancelled_error_propagates(self, mock_extract_file, isolated_env): """CancelledError in _process_one is re-raised, not swallowed.""" @@ -920,7 +924,7 @@ async def _cancel(*args, **kwargs): await ingest_batch([entry], added, {}, {}, {}, quiet=True) @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_task_cancelled_error_does_not_orphan_siblings( self, mock_extract_file, isolated_env @@ -1316,7 +1320,9 @@ def _spy_plan(*args, **kwargs): monkeypatch.setattr(pipeline, "_plan_file_changes", _spy_plan) with mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", + new_callable=mock.AsyncMock, + return_value=_make_kreuzberg_result(), ): await sync(quiet=True) assert plan_threads @@ -1331,7 +1337,9 @@ async def test_sync_backfills_stats_via_store(self, isolated_env, mock_svc): mock_svc.store.upsert_source("legacy.txt", file_hash(f), 1, source_type="document") with mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", + new_callable=mock.AsyncMock, + return_value=_make_kreuzberg_result(), ): result = await sync(quiet=True) assert result.unchanged == 1 @@ -1906,7 +1914,7 @@ def test_paginated_has_page_config(self): from lilbee.data.ingest import ExtractMode, extraction_config config = extraction_config(ExtractMode.PAGINATED) - assert config.pages is not None + assert config.get("pages") is not None def test_paginated_no_markdown_output(self): from lilbee.data.ingest import ExtractMode, extraction_config @@ -1918,27 +1926,28 @@ def test_markdown_has_no_page_config(self): from lilbee.data.ingest import ExtractMode, extraction_config config = extraction_config(ExtractMode.MARKDOWN) - assert config.pages is None + assert config.get("pages") is None def test_markdown_has_chunking(self): from lilbee.data.ingest import ExtractMode, extraction_config config = extraction_config(ExtractMode.MARKDOWN) - assert config.chunking is not None + assert config.get("chunking") is not None def test_markdown_sets_output_format(self): from lilbee.data.ingest import ExtractMode, extraction_config config = extraction_config(ExtractMode.MARKDOWN) - assert config.output_format == "markdown" + assert config["output_format"] == "markdown" - def test_paginated_ocr_has_tesseract(self): + def test_paginated_has_tesseract_ocr_backend(self): from lilbee.data.ingest import ExtractMode, extraction_config - config = extraction_config(ExtractMode.PAGINATED_OCR) - assert config.pages is not None - assert config.ocr is not None - assert config.ocr.backend == "tesseract" + config = extraction_config(ExtractMode.PAGINATED) + assert config.get("pages") is not None + assert config.get("ocr") is not None + # No vision model configured -> kreuzberg's tesseract backend OCRs scanned pages. + assert config["ocr"].backend == "tesseract" @pytest.mark.parametrize( "content_type, expected_mode_name", @@ -1949,7 +1958,7 @@ def test_paginated_ocr_has_tesseract(self): ("xlsx", "MARKDOWN"), ("pptx", "MARKDOWN"), ("epub", "MARKDOWN"), - ("image", "MARKDOWN"), + ("image", "PAGINATED"), ("code", "MARKDOWN"), ], ) @@ -1967,8 +1976,8 @@ def test_topic_threshold_propagates_to_every_mode(self, monkeypatch): monkeypatch.setattr(cfg, "topic_threshold", 0.42) for mode in ExtractMode: config = extraction_config(mode) - assert config.chunking.chunker_type == "semantic" - assert config.chunking.topic_threshold == pytest.approx(0.42, abs=1e-5) + assert config["chunking"].chunker_type == "semantic" + assert config["chunking"].topic_threshold == pytest.approx(0.42, abs=1e-5) class TestClassifyKreuzbergParityFormats: @@ -2022,7 +2031,9 @@ def test_classify(self, filename, expected): assert classify_file(Path(filename)) == expected -@mock.patch("kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result()) +@mock.patch( + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() +) class TestSyncStructuredFormats: async def test_xml_file_ingested( self, @@ -2069,430 +2080,6 @@ async def test_csv_file_ingested( assert "data.csv" in result.added -class TestHasMeaningfulText: - def test_empty_content_and_chunks_returns_false(self): - from lilbee.data.ingest.extract import _has_meaningful_text - - result = mock.MagicMock(content="", chunks=[]) - assert _has_meaningful_text(result) is False - - def test_short_text_returns_false(self): - from lilbee.data.ingest.extract import _has_meaningful_text - - chunk = mock.MagicMock() - chunk.content = "short" - result = mock.MagicMock(content="short", chunks=[chunk]) - assert _has_meaningful_text(result) is False - - def test_meaningful_chunks_returns_true(self): - from lilbee.data.ingest.extract import _has_meaningful_text - - chunk = mock.MagicMock() - chunk.content = "A" * 100 - result = mock.MagicMock(content="", chunks=[chunk]) - assert _has_meaningful_text(result) is True - - def test_content_rich_empty_chunks_returns_true(self): - """A text PDF whose extractor populated content but no chunks must - take the text path, not vision OCR (bb-4u58).""" - from lilbee.data.ingest.extract import _has_meaningful_text - - result = mock.MagicMock(content="A" * 100, chunks=[]) - assert _has_meaningful_text(result) is True - - -def _vision_pages(*pairs): - """Build PageText-like NamedTuples for stubbing provider.pdf_ocr.""" - from lilbee.vision import PageText - - return [PageText(page, text) for page, text in pairs] - - -class TestVisionFallback: - """OCR fallback dispatches to ``provider.pdf_ocr`` (the persistent worker).""" - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_vision_fallback_called_for_empty_pdf(self, mock_kf, isolated_env, mock_svc): - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - cfg.ocr_timeout = 45.0 - cfg.enable_ocr = True - mock_kf.return_value = _make_empty_result() - mock_svc.provider.pdf_ocr.return_value = _vision_pages((1, "Vision extracted text. " * 10)) - - f = isolated_env / "scanned.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - result = await ingest_document(f, "scanned.pdf", "pdf", quiet=True) - mock_svc.provider.pdf_ocr.assert_called_once_with( - f, - backend="vision", - model="org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf", - per_page_timeout_s=45.0, - quiet=True, - on_progress=mock.ANY, - ) - assert len(result) > 0 - assert result[0]["content_type"] == "pdf" - assert result[0]["page_start"] == 1 - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_vision_fallback_quiet_false_by_default(self, mock_kf, isolated_env, mock_svc): - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - cfg.ocr_timeout = 120.0 - cfg.enable_ocr = True - mock_kf.return_value = _make_empty_result() - mock_svc.provider.pdf_ocr.return_value = _vision_pages((1, "Vision extracted text. " * 10)) - - f = isolated_env / "scanned.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - await ingest_document(f, "scanned.pdf", "pdf") - mock_svc.provider.pdf_ocr.assert_called_once_with( - f, - backend="vision", - model="org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf", - per_page_timeout_s=120.0, - quiet=False, - on_progress=mock.ANY, - ) - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_ocr_disabled_skips_both_backends(self, mock_kf, isolated_env, mock_svc): - """``cfg.enable_ocr=False`` disables OCR entirely (bb-ziks.20): neither the - vision pool nor the Tesseract fallback runs. Only the initial text-layer - extract is attempted; finding none, the file is skipped without paying the - Tesseract cost the config says is turned off.""" - mock_kf.return_value = _make_empty_result() - cfg.enable_ocr = False - f = isolated_env / "scanned.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - with mock.patch("lilbee.data.ingest.extract._run_tesseract_sync") as mock_tess: - result = await ingest_document(f, "scanned.pdf", "pdf") - mock_svc.provider.pdf_ocr.assert_not_called() - mock_tess.assert_not_called() - # Only the initial text-layer extract ran; no Tesseract re-extract. - assert mock_kf.call_count == 1 - assert result == [] - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_vision_fallback_not_called_for_non_pdf(self, mock_kf, isolated_env, mock_svc): - mock_kf.return_value = _make_empty_result() - cfg.enable_ocr = True - f = isolated_env / "doc.txt" - f.write_text("") - - from lilbee.data.ingest import ingest_document - - result = await ingest_document(f, "doc.txt", "text") - mock_svc.provider.pdf_ocr.assert_not_called() - assert result == [] - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_vision_fallback_empty_vision_text_returns_empty( - self, mock_kf, isolated_env, mock_svc - ): - cfg.enable_ocr = True - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - mock_kf.return_value = _make_empty_result() - mock_svc.provider.pdf_ocr.return_value = [] - - f = isolated_env / "blank.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - result = await ingest_document(f, "blank.pdf", "pdf") - assert result == [] - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_no_vision_fallback_when_text_meaningful(self, mock_kf, isolated_env, mock_svc): - mock_kf.return_value = _make_kreuzberg_result( - text="Meaningful PDF content. " * 20, num_chunks=1, has_pages=True - ) - cfg.enable_ocr = True - f = isolated_env / "good.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - result = await ingest_document(f, "good.pdf", "pdf") - mock_svc.provider.pdf_ocr.assert_not_called() - assert len(result) > 0 - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_vision_fallback_no_chunks_returns_empty(self, mock_kf, isolated_env, mock_svc): - cfg.enable_ocr = True - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - mock_kf.return_value = _make_empty_result() - mock_svc.provider.pdf_ocr.return_value = _vision_pages((1, "Some text")) - - f = isolated_env / "nochunks.pdf" - f.write_bytes(b"fake pdf") - - with mock.patch("lilbee.data.ingest.extract.chunk_text", return_value=[]): - from lilbee.data.ingest import ingest_document - - result = await ingest_document(f, "nochunks.pdf", "pdf") - assert result == [] - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_vision_fallback_bypasses_semantic_chunking( - self, mock_kf, isolated_env, mock_svc - ): - """Per-page OCR text gets ``chunk_text(use_semantic=False)`` to skip the - embedding round-trip per single-topic page.""" - cfg.enable_ocr = True - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - mock_kf.return_value = _make_empty_result() - mock_svc.provider.pdf_ocr.return_value = _vision_pages( - (1, "page one text"), (2, "page two text") - ) - - f = isolated_env / "bypass.pdf" - f.write_bytes(b"fake pdf") - - with mock.patch( - "lilbee.data.ingest.extract.chunk_text", return_value=["chunk"] - ) as mock_chunk: - from lilbee.data.ingest import ingest_document - - await ingest_document(f, "bypass.pdf", "pdf") - - assert mock_chunk.call_count == 2, "chunk_text called once per OCR page" - for call in mock_chunk.call_args_list: - assert call.kwargs.get("use_semantic") is False - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_vision_cancel_propagates_not_swallowed(self, mock_kf, isolated_env, mock_svc): - """A user cancel raised through the per-page on_progress (TaskCancelledError) - must abort the file, not be logged as an OCR failure and swallowed as [].""" - from lilbee.runtime.cancellation import TaskCancelledError - - cfg.enable_ocr = True - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - mock_kf.return_value = _make_empty_result() - mock_svc.provider.pdf_ocr.side_effect = TaskCancelledError - - f = isolated_env / "cancel.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - with pytest.raises(TaskCancelledError): - await ingest_document(f, "cancel.pdf", "pdf", quiet=True) - - -def _write_png(path: Path) -> None: - """Write a tiny valid PNG so the image OCR path can open + re-encode it.""" - from PIL import Image - - Image.new("RGB", (8, 8), color=(255, 255, 255)).save(path, format="PNG") - - -class TestImageOcr: - """bb-657: image content_type routes through OCR (vision when configured, else Tesseract).""" - - async def test_image_routed_to_vision_ocr(self, isolated_env, mock_svc): - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - cfg.enable_ocr = True - cfg.ocr_timeout = 30.0 - mock_svc.provider.vision_ocr.return_value = "Scanned image text. " * 20 - - f = isolated_env / "scan.png" - _write_png(f) - - from lilbee.data.ingest import ingest_document - - result = await ingest_document(f, "scan.png", "image") - # the single-image path is used, not the PDF page loop - mock_svc.provider.vision_ocr.assert_called_once() - mock_svc.provider.pdf_ocr.assert_not_called() - assert len(result) > 0 - assert result[0]["content_type"] == "image" - assert result[0]["page_start"] == 1 - - async def test_image_skips_kreuzberg_markdown_extract(self, isolated_env, mock_svc): - # The pre-fix bug: an image went through a markdown extract that yields no - # text. The image branch must not call kreuzberg.extract_file_sync at all. - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - cfg.enable_ocr = True - mock_svc.provider.vision_ocr.return_value = "text " * 20 - f = isolated_env / "scan.png" - _write_png(f) - - from lilbee.data.ingest import ingest_document - - with mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) as mock_kf: - await ingest_document(f, "scan.png", "image") - mock_kf.assert_not_called() - - async def test_image_falls_back_to_tesseract_without_vision_model(self, isolated_env, mock_svc): - cfg.vision_model = "" - cfg.enable_ocr = None # auto: no vision model -> Tesseract - f = isolated_env / "scan.png" - _write_png(f) - - from lilbee.data.ingest import ingest_document - - with mock.patch("lilbee.data.ingest.extract._run_tesseract_sync") as mock_tess: - mock_tess.return_value = _make_kreuzberg_result("Tesseract image text. " * 20) - result = await ingest_document(f, "scan.png", "image") - mock_svc.provider.vision_ocr.assert_not_called() - assert len(result) > 0 - assert result[0]["content_type"] == "image" - - async def test_image_vision_ocr_failure_skips_file(self, isolated_env, mock_svc): - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - cfg.enable_ocr = True - mock_svc.provider.vision_ocr.side_effect = RuntimeError("vision down") - f = isolated_env / "scan.png" - _write_png(f) - - from lilbee.data.ingest import ingest_document - - assert await ingest_document(f, "scan.png", "image") == [] - - async def test_image_ocr_disabled_skips_both_backends(self, isolated_env, mock_svc): - """enable_ocr=False disables OCR entirely for images (bb-ziks.20): neither - the vision pool nor the Tesseract fallback runs.""" - cfg.enable_ocr = False - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - f = isolated_env / "scan.png" - _write_png(f) - - from lilbee.data.ingest import ingest_document - - with mock.patch("lilbee.data.ingest.extract._run_tesseract_sync") as mock_tess: - result = await ingest_document(f, "scan.png", "image") - assert result == [] - mock_svc.provider.vision_ocr.assert_not_called() - mock_tess.assert_not_called() - - async def test_vision_ocr_cache_key_includes_timeout(self, isolated_env, mock_svc, monkeypatch): - """The vision OCR cache key carries the per-page timeout so raising it - re-OCRs rather than serving an earlier partially-empty result for the same - content (bb-ziks.39). Also exercises the cache-hit reuse path.""" - from lilbee.data.ingest import extract - from lilbee.runtime.progress import noop_callback - - cfg.enable_ocr = True - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - cfg.ocr_timeout = 77.0 - captured: dict[str, str] = {} - - def fake_key(file_h, *, backend, model, extra): - captured["extra"] = extra - return "cache-key" - - monkeypatch.setattr(extract, "ocr_cache_key", fake_key) - monkeypatch.setattr(extract, "load_ocr_pages", lambda key: [(1, "cached page text")]) - f = isolated_env / "scan.png" - _write_png(f) - - result = await extract._vision_image_ocr(f, "scan.png", "image", on_progress=noop_callback) - assert "77" in captured["extra"] - assert result # cache hit produced chunks from the cached page, no re-OCR - mock_svc.provider.vision_ocr.assert_not_called() - - async def test_image_tesseract_empty_skips_file(self, isolated_env, mock_svc): - cfg.vision_model = "" - cfg.enable_ocr = None - f = isolated_env / "scan.png" - _write_png(f) - - from lilbee.data.ingest import ingest_document - - with mock.patch("lilbee.data.ingest.extract._run_tesseract_sync") as mock_tess: - mock_tess.return_value = _make_empty_result() - assert await ingest_document(f, "scan.png", "image") == [] - - async def test_multipage_image_ocrs_every_frame_end_to_end(self, isolated_env, mock_svc): - """A multi-frame TIFF routed to vision OCR yields one OCR call and one page - record per frame (bb-7jg1.10), not just the first frame.""" - cfg.enable_ocr = True - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - mock_svc.provider.vision_ocr.return_value = "frame text " * 5 - from PIL import Image - - f = isolated_env / "multi.tiff" - frames = [Image.new("RGB", (4, 4), color=c) for c in [(1, 2, 3), (4, 5, 6), (7, 8, 9)]] - frames[0].save(f, format="TIFF", save_all=True, append_images=frames[1:]) - - from lilbee.data.ingest import ingest_document - - records = await ingest_document(f, "multi.tiff", "image") - assert mock_svc.provider.vision_ocr.call_count == 3 - assert sorted({r["page_start"] for r in records}) == [1, 2, 3] - - def test_image_page_pngs_single_frame_reencodes(self, isolated_env): - from lilbee.data.ingest.extract import _image_page_pngs - - f = isolated_env / "x.bmp" - from PIL import Image - - Image.new("RGB", (4, 4), color=(1, 2, 3)).save(f, format="BMP") - pages = _image_page_pngs(f) - assert len(pages) == 1 - assert pages[0].startswith(b"\x89PNG\r\n") - - def test_image_page_pngs_multipage_tiff_yields_all_frames(self, isolated_env): - """A multi-frame TIFF produces one PNG page per frame; the prior code - dropped every frame after the first (bb-7jg1.10).""" - from lilbee.data.ingest.extract import _image_page_pngs - - f = isolated_env / "multi.tiff" - from PIL import Image - - frame0 = Image.new("RGB", (4, 4), color=(1, 2, 3)) - frame1 = Image.new("RGB", (4, 4), color=(4, 5, 6)) - frame2 = Image.new("RGB", (4, 4), color=(7, 8, 9)) - frame0.save(f, format="TIFF", save_all=True, append_images=[frame1, frame2]) - pages = _image_page_pngs(f) - assert len(pages) == 3 - assert all(p.startswith(b"\x89PNG\r\n") for p in pages) - - -class TestShouldRunOcrAutoDetect: - def test_auto_detect_vision_model_set(self, isolated_env): - """When enable_ocr is None, _should_run_ocr reflects cfg.vision_model.""" - cfg.enable_ocr = None - cfg.vision_model = "noctrex/LightOnOCR-2-1B-GGUF/lightonocr-Q4_K_M.gguf" - from lilbee.data.ingest.extract import _should_run_ocr - - assert _should_run_ocr() is True - - def test_auto_detect_vision_model_empty(self, isolated_env): - """When enable_ocr is None and cfg.vision_model is empty, returns False.""" - cfg.enable_ocr = None - cfg.vision_model = "" - from lilbee.data.ingest.extract import _should_run_ocr - - assert _should_run_ocr() is False - - def test_force_on_with_empty_vision_model(self, isolated_env): - """enable_ocr=True still returns True; caller may fall back to Tesseract.""" - cfg.enable_ocr = True - cfg.vision_model = "" - from lilbee.data.ingest.extract import _should_run_ocr - - assert _should_run_ocr() is True - - def test_force_off(self, isolated_env): - """enable_ocr=False always returns False regardless of vision_model.""" - cfg.enable_ocr = False - cfg.vision_model = "noctrex/LightOnOCR-2-1B-GGUF/lightonocr-Q4_K_M.gguf" - from lilbee.data.ingest.extract import _should_run_ocr - - assert _should_run_ocr() is False - - class TestOcrOverrideContextVar: """Per-request OCR overrides use a ContextVar, never a global cfg mutation.""" @@ -2528,14 +2115,6 @@ def test_none_arguments_keep_cfg_defaults(self, isolated_env): with ocr_override(enable_ocr=None, ocr_timeout=None): assert _effective_enable_ocr() is True - def test_should_run_ocr_honors_override(self, isolated_env): - from lilbee.data.ingest.extract import _should_run_ocr, ocr_override - - cfg.enable_ocr = True - cfg.vision_model = "" - with ocr_override(enable_ocr=False): - assert _should_run_ocr() is False - def test_overrides_isolated_across_contexts(self, isolated_env): # Two copied contexts must each see only their own override; this is the # concurrency guarantee that a global cfg mutation could not give. @@ -2579,157 +2158,6 @@ async def test_override_propagates_into_to_thread_worker(self, isolated_env): assert await asyncio.to_thread(_effective_ocr_timeout) == 5.0 -class TestOcrFallbackBackendDispatch: - """``_handle_scanned_pdf_fallback`` routes to the right backend on the pool.""" - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_vision_backend_when_ocr_enabled_and_model_set( - self, mock_kf, isolated_env, mock_svc - ): - cfg.enable_ocr = True - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - cfg.ocr_timeout = 60.0 - mock_kf.return_value = _make_empty_result() - mock_svc.provider.pdf_ocr.return_value = _vision_pages((1, "Vision text. " * 10)) - - f = isolated_env / "scanned.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - result = await ingest_document(f, "scanned.pdf", "pdf") - assert mock_svc.provider.pdf_ocr.call_args.kwargs["backend"] == "vision" - assert mock_svc.provider.pdf_ocr.call_args.kwargs["per_page_timeout_s"] == 60.0 - assert len(result) > 0 - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_tesseract_backend_when_no_vision_model(self, mock_kf, isolated_env, mock_svc): - """Without a vision model the scanned PDF runs Tesseract inline (no pool).""" - cfg.enable_ocr = True - cfg.vision_model = "" - cfg.tesseract_timeout = 30.0 - # First call: pre-extract that flags the PDF as scanned. Second - # call: the inline Tesseract fallback that produces real chunks. - mock_kf.side_effect = [ - _make_empty_result(), - _make_kreuzberg_result(text="Tesseract OCR text. " * 10, num_chunks=1, has_pages=True), - ] - - f = isolated_env / "scanned.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - result = await ingest_document(f, "scanned.pdf", "pdf") - # Pool pdf_ocr is not used for tesseract. - mock_svc.provider.pdf_ocr.assert_not_called() - assert mock_kf.call_count == 2 - assert len(result) > 0 - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_tesseract_returning_empty_logs_skip( - self, mock_kf, isolated_env, mock_svc, caplog - ): - """Tesseract returning no pages produces a user-facing skip warning.""" - cfg.enable_ocr = None # auto: no vision model -> Tesseract fallback runs - cfg.vision_model = "" - # Both pre-extract and tesseract fallback return empty. - mock_kf.side_effect = [_make_empty_result(), _make_empty_result()] - - f = isolated_env / "blank.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - with caplog.at_level("WARNING", logger="lilbee.data.ingest.extract"): - result = await ingest_document(f, "blank.pdf", "pdf") - assert result == [] - assert "Skipped blank.pdf" in caplog.text - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_pool_exception_returns_empty(self, mock_kf, isolated_env, mock_svc): - """Worker error in pdf_ocr is logged and surfaces as an empty result.""" - cfg.enable_ocr = True - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - mock_kf.return_value = _make_empty_result() - mock_svc.provider.pdf_ocr.side_effect = RuntimeError("pool died") - - f = isolated_env / "broken.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - result = await ingest_document(f, "broken.pdf", "pdf", quiet=True) - assert result == [] - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_tesseract_no_timeout_runs_uncapped(self, mock_kf, isolated_env, mock_svc): - """``cfg.tesseract_timeout == 0`` skips ``asyncio.wait_for`` and runs uncapped.""" - cfg.enable_ocr = None # auto: no vision model -> Tesseract fallback runs - cfg.vision_model = "" - cfg.tesseract_timeout = 0 - mock_kf.side_effect = [ - _make_empty_result(), - _make_kreuzberg_result(text="Tesseract OCR text. " * 10, num_chunks=1, has_pages=True), - ] - - f = isolated_env / "scanned.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - result = await ingest_document(f, "scanned.pdf", "pdf") - assert mock_kf.call_count == 2 - assert len(result) > 0 - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_tesseract_timeout_logs_and_returns_empty( - self, mock_kf, isolated_env, mock_svc, caplog - ): - """Tesseract exceeding the wall-clock cap logs a warning and skips the file.""" - cfg.enable_ocr = None # auto: no vision model -> Tesseract fallback runs - cfg.vision_model = "" - cfg.tesseract_timeout = 30.0 - # Pre-extract returns empty; tesseract fallback's wait_for fires. - mock_kf.side_effect = [_make_empty_result(), _make_kreuzberg_result()] - - async def _instant_timeout(coro, *, timeout): - coro.close() - raise TimeoutError - - with mock.patch("asyncio.wait_for", side_effect=_instant_timeout): - f = isolated_env / "scanned.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - with caplog.at_level("WARNING", logger="lilbee.data.ingest.extract"): - result = await ingest_document(f, "scanned.pdf", "pdf") - assert result == [] - assert "Tesseract OCR exceeded" in caplog.text - - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_tesseract_extract_exception_logs_and_returns_empty( - self, mock_kf, isolated_env, mock_svc, caplog - ): - """A kreuzberg backend crash logs a warning and returns no chunks.""" - cfg.enable_ocr = None # auto: no vision model -> Tesseract fallback runs - cfg.vision_model = "" - cfg.tesseract_timeout = 30.0 - # Pre-extract OK; tesseract fallback raises. - mock_kf.side_effect = [_make_empty_result(), RuntimeError("tesseract binary missing")] - - f = isolated_env / "scanned.pdf" - f.write_bytes(b"fake pdf") - - from lilbee.data.ingest import ingest_document - - with caplog.at_level("WARNING", logger="lilbee.data.ingest.extract"): - result = await ingest_document(f, "scanned.pdf", "pdf") - assert result == [] - assert "OCR via tesseract backend failed" in caplog.text - - class TestPhaseProgressCallback: """The non-quiet Rich bar advances once per file, so a single multi-page file would freeze at "0/1" through its whole OCR + embed phase. The phase-progress @@ -2814,7 +2242,7 @@ async def test_frontmatter_only_produces_chunks(self, isolated_env): class TestPageTextAccumulator: """`page_texts_out` captures clean per-page text for the export dataset.""" - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) + @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) async def test_pdf_pages_captured(self, mock_kf, isolated_env): mock_kf.return_value = _make_kreuzberg_result(num_chunks=2, has_pages=True) from lilbee.data.ingest import ingest_document @@ -2826,7 +2254,7 @@ async def test_pdf_pages_captured(self, mock_kf, isolated_env): assert [p["page"] for p in pages] == [1, 2] assert all(p["content_type"] == "pdf" for p in pages) - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) + @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) async def test_non_paginated_doc_captured_as_page_zero(self, mock_kf, isolated_env): mock_kf.return_value = _make_kreuzberg_result(text="Plain body. " * 10, has_pages=False) from lilbee.data.ingest import ingest_document @@ -2851,15 +2279,14 @@ async def test_markdown_captured_as_page_zero(self, isolated_env): assert pages[0]["page"] == 0 assert "markdown body" in pages[0]["text"] - @mock.patch("kreuzberg.extract_file_sync", new_callable=Mock) - async def test_ocr_fallback_captured(self, mock_kf, isolated_env, mock_svc): + @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) + async def test_ocr_pages_captured(self, mock_kf, isolated_env, mock_svc): + # kreuzberg OCRs scanned pages in-pass; the OCR'd text arrives in result.pages. cfg.enable_ocr = True cfg.vision_model = "" - cfg.tesseract_timeout = 30.0 - mock_kf.side_effect = [ - _make_empty_result(), - _make_kreuzberg_result(text="OCR page text. " * 10, num_chunks=2, has_pages=True), - ] + mock_kf.return_value = _make_kreuzberg_result( + text="OCR page text. " * 10, num_chunks=2, has_pages=True + ) from lilbee.data.ingest import ingest_document f = isolated_env / "scanned.pdf" @@ -2876,8 +2303,8 @@ async def test_empty_extraction_returns_empty(self, isolated_env): from lilbee.data.ingest import ingest_document empty_result = mock.MagicMock(chunks=[]) - mock_extract = Mock(return_value=empty_result) - with mock.patch("kreuzberg.extract_file_sync", mock_extract): + mock_extract = mock.AsyncMock(return_value=empty_result) + with mock.patch("kreuzberg.extract_file", mock_extract): result = await ingest_document(isolated_env / "e.xml", "e.xml", "xml") assert result == [] @@ -2885,8 +2312,8 @@ async def test_no_chunks_returns_empty(self, isolated_env): from lilbee.data.ingest import ingest_document no_chunks_result = mock.MagicMock(chunks=[]) - mock_extract = Mock(return_value=no_chunks_result) - with mock.patch("kreuzberg.extract_file_sync", mock_extract): + mock_extract = mock.AsyncMock(return_value=no_chunks_result) + with mock.patch("kreuzberg.extract_file", mock_extract): result = await ingest_document(isolated_env / "s.xml", "s.xml", "xml") assert result == [] @@ -2911,7 +2338,7 @@ def _mock_concepts_available(self): yield @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_concept_extraction_called_during_ingest( self, mock_extract_file, isolated_env, mock_svc @@ -2928,7 +2355,7 @@ async def test_concept_extraction_called_during_ingest( mock_svc.concepts.extract_concepts_batch.assert_called() @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_concept_disabled_skips_extraction( self, mock_extract_file, isolated_env, mock_svc @@ -2943,7 +2370,7 @@ async def test_concept_disabled_skips_extraction( mock_svc.concepts.extract_concepts_batch.assert_not_called() @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_concept_failure_does_not_break_ingest( self, mock_extract_file, isolated_env, mock_svc @@ -2960,7 +2387,7 @@ async def test_concept_failure_does_not_break_ingest( assert "concept_test2.txt" in result.added @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_concept_write_failure_does_not_fail_files( self, mock_extract_file, isolated_env, mock_svc @@ -2982,7 +2409,7 @@ async def test_concept_write_failure_does_not_fail_files( assert result.failed == [] @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_cluster_rebuild_called_after_sync( self, mock_extract_file, isolated_env, mock_svc @@ -2999,7 +2426,7 @@ async def test_cluster_rebuild_called_after_sync( mock_svc.concepts.rebuild_clusters.assert_called() @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_cluster_rebuild_failure_does_not_break_sync( self, mock_extract_file, isolated_env, mock_svc @@ -3017,7 +2444,7 @@ async def test_cluster_rebuild_failure_does_not_break_sync( assert "rebuild_test.txt" in result.added @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_graph_none_skips_indexing(self, mock_extract_file, isolated_env, mock_svc): """When get_graph() returns None, concept indexing is skipped gracefully.""" @@ -3031,7 +2458,7 @@ async def test_graph_none_skips_indexing(self, mock_extract_file, isolated_env, assert "graph_none_test.txt" in result.added @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_concepts_unavailable_skips_rebuild( self, mock_extract_file, isolated_env, mock_svc @@ -3048,7 +2475,7 @@ async def test_concepts_unavailable_skips_rebuild( mock_svc.concepts.rebuild_clusters.assert_not_called() @mock.patch( - "kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result() + "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() ) async def test_concepts_unavailable_skips_indexing( self, mock_extract_file, isolated_env, mock_svc @@ -3115,3 +2542,80 @@ def test_no_marker_when_nothing_removed(self, isolated_env, mock_svc): mock_svc.store.remove_documents.return_value = RemoveResult(removed=[], not_found=["gone"]) remove_documents_durably(["gone"]) assert load_skip_markers(cfg.data_root) == {} + + +class TestOcrConfigSelection: + """extraction_config picks the OCR backend from enable_ocr + vision_model.""" + + def test_disabled_when_enable_ocr_false(self, isolated_env): + from lilbee.data.ingest import ExtractMode, extraction_config + + cfg.enable_ocr = False + config = extraction_config(ExtractMode.PAGINATED) + assert config["ocr"].enabled is False + + def test_vision_backend_with_token_when_model_set(self, isolated_env): + from lilbee.data.ingest import ExtractMode, extraction_config + + cfg.enable_ocr = None + cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" + config = extraction_config(ExtractMode.PAGINATED, ocr_token="tok-123") + assert config["ocr"].backend == "lilbee-vision" + assert "tok-123" in config["ocr"].backend_options + + +class TestChunkAndEmbedPagesEmpty: + async def test_empty_page_texts_returns_empty(self): + from lilbee.data.ingest.extract import chunk_and_embed_pages + from lilbee.runtime.progress import noop_callback + + assert await chunk_and_embed_pages([], "s", "pdf", noop_callback) == [] + + async def test_whitespace_pages_yield_no_chunks(self, mock_svc): + from lilbee.data.ingest.extract import chunk_and_embed_pages + from lilbee.runtime.progress import noop_callback + + assert await chunk_and_embed_pages([(1, " ")], "s", "pdf", noop_callback) == [] + + +class TestIngestDocumentOcrPath: + @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) + async def test_empty_pdf_warns_no_usable_text(self, mock_kf, isolated_env, mock_svc, caplog): + mock_kf.return_value = mock.MagicMock(chunks=[]) + from lilbee.data.ingest import ingest_document + + f = isolated_env / "scan.pdf" + f.write_bytes(b"x") + result = await ingest_document(f, "scan.pdf", "pdf") + assert result == [] + assert "no usable text" in caplog.text + + @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) + async def test_streams_per_page_progress_ticks(self, mock_kf, isolated_env, mock_svc): + import json + + cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" + result_obj = _make_kreuzberg_result(num_chunks=1, has_pages=True) + + async def fake_extract(path, *, config): + # Simulate kreuzberg calling the registered backend once per scanned page. + from lilbee.data.ingest.vision_ocr_backend import ocr_requests + + token = json.loads(config["ocr"].backend_options)["req"] + ctx = ocr_requests.get(token) + ctx.on_page() + ctx.on_page() + return result_obj + + mock_kf.side_effect = fake_extract + from lilbee.data.ingest import ingest_document + + seen: list[int] = [] + + def on_prog(_et, ev): + seen.append(ev.page) + + f = isolated_env / "scan.pdf" + f.write_bytes(b"x") + await ingest_document(f, "scan.pdf", "pdf", on_progress=on_prog) + assert 1 in seen and 2 in seen # two per-page running-count ticks diff --git a/tests/test_ocr_cache.py b/tests/test_ocr_cache.py deleted file mode 100644 index ea9b82018..000000000 --- a/tests/test_ocr_cache.py +++ /dev/null @@ -1,125 +0,0 @@ -"""OCR output is cached by file content + backend/model so a downstream failure -doesn't discard an expensive OCR pass on retry.""" - -from __future__ import annotations - -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from lilbee.core.config import cfg -from lilbee.data.ingest import extract, ocr_cache -from lilbee.vision import PageText - - -@pytest.fixture -def cache_dir(tmp_path: Path, monkeypatch) -> Path: - data_dir = tmp_path / "data" - monkeypatch.setattr(cfg, "data_dir", data_dir) - return data_dir / "ocr_cache" - - -def test_key_is_stable_and_varies_by_component() -> None: - base = ocr_cache.ocr_cache_key("hash1", backend="vision", model="m") - assert base == ocr_cache.ocr_cache_key("hash1", backend="vision", model="m") - assert base != ocr_cache.ocr_cache_key("hash2", backend="vision", model="m") - assert base != ocr_cache.ocr_cache_key("hash1", backend="tesseract", model="m") - assert base != ocr_cache.ocr_cache_key("hash1", backend="vision", model="other") - assert base != ocr_cache.ocr_cache_key("hash1", backend="vision", model="m", extra="2048") - - -def test_store_then_load_round_trips(cache_dir: Path) -> None: - pages = [(1, "page one"), (2, "page two")] - key = ocr_cache.ocr_cache_key("h", backend="vision", model="m") - ocr_cache.store_ocr_pages(key, pages) - assert (cache_dir / f"{key}.json").exists() - assert ocr_cache.load_ocr_pages(key) == pages - - -def test_load_returns_none_on_miss(cache_dir: Path) -> None: - assert ocr_cache.load_ocr_pages("nope") is None - - -def test_load_returns_none_on_corrupt_entry(cache_dir: Path) -> None: - cache_dir.mkdir(parents=True) - (cache_dir / "bad.json").write_text("{not json", encoding="utf-8") - assert ocr_cache.load_ocr_pages("bad") is None - - -def test_empty_pages_are_not_cached(cache_dir: Path) -> None: - key = ocr_cache.ocr_cache_key("h", backend="tesseract", model="tesseract") - ocr_cache.store_ocr_pages(key, []) - assert ocr_cache.load_ocr_pages(key) is None - - -def test_load_returns_none_when_json_is_not_a_list(cache_dir: Path) -> None: - # Valid JSON of the wrong shape (a dict, not a list of pairs) is still a miss. - cache_dir.mkdir(parents=True) - (cache_dir / "wrong.json").write_text('{"page": 1}', encoding="utf-8") - assert ocr_cache.load_ocr_pages("wrong") is None - - -def test_store_swallows_write_error(tmp_path: Path, monkeypatch) -> None: - # data_dir points at a regular file, so creating the cache dir under it raises - # OSError; store must log and swallow rather than break ingestion. - data_file = tmp_path / "data" - data_file.write_text("not a directory", encoding="utf-8") - monkeypatch.setattr(cfg, "data_dir", data_file) - key = ocr_cache.ocr_cache_key("h", backend="vision", model="m") - ocr_cache.store_ocr_pages(key, [(1, "page")]) # must not raise - - -@pytest.mark.asyncio -async def test_vision_fallback_uses_cache_and_skips_ocr(tmp_path: Path, monkeypatch) -> None: - # A cache hit must short-circuit the expensive pdf_ocr call and feed the - # cached pages straight into chunk + embed. - pdf = tmp_path / "scan.pdf" - pdf.write_bytes(b"%PDF-1.4 fake") - monkeypatch.setattr(extract, "load_ocr_pages", lambda _k: [(1, "cached page")]) - - def _boom(*_a, **_k): - raise AssertionError("pdf_ocr must not run on a cache hit") - - monkeypatch.setattr( - extract, "get_services", lambda: SimpleNamespace(provider=SimpleNamespace(pdf_ocr=_boom)) - ) - - seen: dict[str, object] = {} - - async def _capture(page_texts, *_a, **_k): - seen["pages"] = list(page_texts) - return [] - - monkeypatch.setattr(extract, "chunk_and_embed_pages", _capture) - out = await extract._vision_ocr_fallback( - pdf, "scan.pdf", "pdf", on_progress=lambda *_a, **_k: None, quiet=True - ) - assert out == [] - assert seen["pages"] == [(1, "cached page")] - - -@pytest.mark.asyncio -async def test_vision_fallback_stores_fresh_ocr(tmp_path: Path, monkeypatch) -> None: - # On a miss, OCR runs and its per-page output is cached for the next retry. - pdf = tmp_path / "scan.pdf" - pdf.write_bytes(b"%PDF-1.4 fake") - monkeypatch.setattr(extract, "load_ocr_pages", lambda _k: None) - monkeypatch.setattr( - extract, - "get_services", - lambda: SimpleNamespace( - provider=SimpleNamespace(pdf_ocr=lambda *_a, **_k: [PageText(1, "fresh")]) - ), - ) - stored: dict[str, object] = {} - monkeypatch.setattr(extract, "store_ocr_pages", lambda _k, pages: stored.update(pages=pages)) - - async def _noop(_page_texts, *_a, **_k): - return [] - - monkeypatch.setattr(extract, "chunk_and_embed_pages", _noop) - await extract._vision_ocr_fallback( - pdf, "scan.pdf", "pdf", on_progress=lambda *_a, **_k: None, quiet=True - ) - assert stored["pages"] == [(1, "fresh")] diff --git a/tests/test_providers.py b/tests/test_providers.py index 8102997fb..730e4ce64 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -3579,65 +3579,6 @@ def test_rerank_with_empty_model_raises_provider_error(self) -> None: rp.rerank("q", ["a", "b"]) -class TestRoutingProviderPdfOcr: - """``RoutingProvider.pdf_ocr`` dispatches by ref prefix, like ``vision_ocr``.""" - - def test_native_ref_routes_to_local_engine(self) -> None: - from lilbee.providers.routing_provider import RoutingProvider - - rp = RoutingProvider() - mock_native = mock.MagicMock() - mock_native.pdf_ocr.return_value = ["p1", "p2"] - rp._local = mock_native - progress = mock.MagicMock() - native_ref = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - - result = rp.pdf_ocr( - Path("/x.pdf"), - backend="vision", - model=native_ref, - per_page_timeout_s=12.5, - quiet=False, - on_progress=progress, - ) - - assert result == ["p1", "p2"] - mock_native.pdf_ocr.assert_called_once_with( - Path("/x.pdf"), - backend="vision", - model=native_ref, - per_page_timeout_s=12.5, - quiet=False, - on_progress=progress, - ) - - def test_hosted_ref_routes_to_sdk_which_raises(self) -> None: - """A hosted ``cfg.vision_model`` reaches the SDK side, which raises.""" - from lilbee.providers.routing_provider import RoutingProvider - - rp = RoutingProvider() - mock_sdk = mock.MagicMock() - mock_sdk.pdf_ocr.side_effect = NotImplementedError("hosted PDF OCR not supported") - rp._sdk_provider = mock_sdk - cfg.vision_model = "" - - with pytest.raises(NotImplementedError): - rp.pdf_ocr(Path("/x.pdf"), backend="vision", model="openai/gpt-4-vision") - mock_sdk.pdf_ocr.assert_called_once() - - -class TestSdkLLMProviderPdfOcr: - """``SdkLLMProvider.pdf_ocr`` cannot rasterise PDFs and must raise.""" - - def test_raises_not_implemented_with_user_facing_message(self) -> None: - from lilbee.providers.litellm_sdk import LitellmSdkBackend - from lilbee.providers.sdk_llm_provider import SdkLLMProvider - - provider = SdkLLMProvider(LitellmSdkBackend()) - with pytest.raises(NotImplementedError, match="LILBEE_VISION_MODEL"): - provider.pdf_ocr(Path("/scan.pdf"), backend="vision") - - class TestChatWithToolsRouting: def test_base_default_raises(self) -> None: from lilbee.providers.base import LLMProvider, ProviderError diff --git a/tests/test_services.py b/tests/test_services.py index 075856cc5..37b3afa1d 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -18,6 +18,72 @@ def isolated_cfg(): setattr(cfg, name, getattr(snapshot, name)) +class TestSyncVisionOcrBackend: + def _patch_kreuzberg(self, monkeypatch, *, listed): + reg = MagicMock() + unreg = MagicMock() + monkeypatch.setattr("kreuzberg.list_ocr_backends", lambda: listed) + monkeypatch.setattr("kreuzberg.register_ocr_backend", reg) + monkeypatch.setattr("kreuzberg.unregister_ocr_backend", unreg) + return reg, unreg + + def test_registers_when_model_set_and_absent(self, monkeypatch): + from lilbee.app.services import sync_vision_ocr_backend + + monkeypatch.setattr(cfg, "vision_model", "vendor/glm-ocr") + reg, unreg = self._patch_kreuzberg(monkeypatch, listed=["tesseract"]) + sync_vision_ocr_backend(MagicMock()) + reg.assert_called_once() + unreg.assert_not_called() + + def test_rebinds_to_current_provider_when_already_registered(self, monkeypatch): + """A rebuilt provider must replace the stale binding: unregister then re-register. + + ``reset_services`` shuts the old provider down; if sync left the prior + registration in place, kreuzberg would keep routing OCR to the dead provider. + """ + from lilbee.app.services import sync_vision_ocr_backend + + monkeypatch.setattr(cfg, "vision_model", "vendor/glm-ocr") + reg, unreg = self._patch_kreuzberg(monkeypatch, listed=["lilbee-vision"]) + sync_vision_ocr_backend(MagicMock()) + unreg.assert_called_once_with("lilbee-vision") + reg.assert_called_once() + + def test_unregisters_when_model_cleared(self, monkeypatch): + from lilbee.app.services import sync_vision_ocr_backend + + monkeypatch.setattr(cfg, "vision_model", "") + reg, unreg = self._patch_kreuzberg(monkeypatch, listed=["lilbee-vision"]) + sync_vision_ocr_backend(MagicMock()) + unreg.assert_called_once_with("lilbee-vision") + reg.assert_not_called() + + def test_noop_when_no_model_and_absent(self, monkeypatch): + from lilbee.app.services import sync_vision_ocr_backend + + monkeypatch.setattr(cfg, "vision_model", "") + reg, unreg = self._patch_kreuzberg(monkeypatch, listed=["tesseract"]) + sync_vision_ocr_backend(MagicMock()) + reg.assert_not_called() + unreg.assert_not_called() + + def test_settings_role_reload_syncs_vision_backend(self, monkeypatch): + """A vision_model change via any settings path (REST/MCP/TUI/CLI) registers it.""" + from lilbee.app.services import set_services + from lilbee.app.settings import _reload_changed_roles + from tests.conftest import make_mock_services + + set_services(make_mock_services()) + try: + monkeypatch.setattr(cfg, "vision_model", "org/V-GGUF/v-Q4_K_M.gguf") + reg, _unreg = self._patch_kreuzberg(monkeypatch, listed=["tesseract"]) + _reload_changed_roles({"vision_model"}) + reg.assert_called_once() + finally: + set_services(None) + + class TestServicesDataclass: def test_fields_are_immutable(self): from lilbee.app.services import CrawlerSyncState, Services diff --git a/tests/test_vision.py b/tests/test_vision.py index 0575d1b20..821d849be 100644 --- a/tests/test_vision.py +++ b/tests/test_vision.py @@ -1,133 +1,19 @@ -"""Tests for vision model OCR extraction.""" +"""Tests for the vision-message helpers (prompt resolution + image messages).""" -from pathlib import Path -from unittest import mock +from __future__ import annotations -import pytest -from lilbee.app.services import CrawlerSyncState, Services, set_services - - -@pytest.fixture() -def mock_provider(): - """Create a mock provider with the full LLMProvider surface. - - ``vision_ocr`` is part of the protocol now, so every concrete provider - implements it; tests mock it directly rather than the chat fallthrough. - """ - provider = mock.MagicMock( - spec=[ - "chat", - "embed", - "vision_ocr", - "list_models", - "pull_model", - "show_model", - "shutdown", - ] - ) - store = mock.MagicMock() - embedder = mock.MagicMock() - reranker = mock.MagicMock() - concepts = mock.MagicMock() - searcher = mock.MagicMock() - registry = mock.MagicMock() - services = Services( - provider=provider, - store=store, - embedder=embedder, - reranker=reranker, - concepts=concepts, - clusterer=mock.MagicMock(), - searcher=searcher, - registry=registry, - hf_client=mock.MagicMock(), - ingest_lock_registry=mock.MagicMock(), - model_manager=mock.MagicMock(), - crawler_semaphore=None, - crawler_sync_state=CrawlerSyncState(), - ) - set_services(services) - yield provider - set_services(None) - - -def _mock_iterator(num_pages: int = 1) -> mock.MagicMock: - """Build a mock PdfPageIterator that yields (index, png_bytes) tuples.""" - pages = [(i, b"\x89PNG" + bytes(f"page-{i}", "utf-8")) for i in range(num_pages)] - it = mock.MagicMock() - it.__len__ = mock.Mock(return_value=num_pages) - it.__iter__ = mock.Mock(return_value=iter(pages)) - it.__enter__ = mock.Mock(return_value=it) - it.__exit__ = mock.Mock(return_value=False) - return it - - -class TestPdfPageCount: - def test_returns_page_count(self) -> None: - mock_iter = _mock_iterator(num_pages=5) - with mock.patch("kreuzberg.PdfPageIterator", return_value=mock_iter): - from lilbee.vision import pdf_page_count - - assert pdf_page_count(Path("test.pdf")) == 5 - - def test_empty_pdf_returns_zero(self) -> None: - mock_iter = _mock_iterator(num_pages=0) - with mock.patch("kreuzberg.PdfPageIterator", return_value=mock_iter): - from lilbee.vision import pdf_page_count - - assert pdf_page_count(Path("empty.pdf")) == 0 - - def test_passes_dpi(self) -> None: - mock_iter = _mock_iterator(num_pages=1) - mock_cls = mock.patch("kreuzberg.PdfPageIterator", return_value=mock_iter) - with mock_cls as patched: - from lilbee.vision import _RASTER_DPI, pdf_page_count - - pdf_page_count(Path("test.pdf")) - patched.assert_called_once_with(mock.ANY, dpi=_RASTER_DPI) - - -class TestRasterizePdf: - def test_yields_index_and_png_bytes(self) -> None: - mock_iter = _mock_iterator(num_pages=2) - with mock.patch("kreuzberg.PdfPageIterator", return_value=mock_iter): - from lilbee.vision import rasterize_pdf - - pages = list(rasterize_pdf(Path("test.pdf"))) - - assert len(pages) == 2 - assert pages[0][0] == 0 - assert pages[1][0] == 1 - assert all(data.startswith(b"\x89PNG") for _, data in pages) - - def test_empty_pdf_yields_nothing(self) -> None: - mock_iter = _mock_iterator(num_pages=0) - with mock.patch("kreuzberg.PdfPageIterator", return_value=mock_iter): - from lilbee.vision import rasterize_pdf - - pages = list(rasterize_pdf(Path("empty.pdf"))) - - assert pages == [] - - def test_uses_context_manager(self) -> None: - mock_iter = _mock_iterator(num_pages=1) - with mock.patch("kreuzberg.PdfPageIterator", return_value=mock_iter): - from lilbee.vision import rasterize_pdf - - list(rasterize_pdf(Path("test.pdf"))) +class TestResolveOcrPrompt: + def test_native_prompt_for_known_family(self) -> None: + from lilbee.vision import resolve_ocr_prompt - mock_iter.__enter__.assert_called_once() - mock_iter.__exit__.assert_called_once() + assert resolve_ocr_prompt("vendor/glm-ocr-1b") == "OCR" + assert resolve_ocr_prompt("deepseek-ocr").startswith("<|grounding|>") - def test_passes_dpi(self) -> None: - mock_iter = _mock_iterator(num_pages=1) - mock_cls = mock.patch("kreuzberg.PdfPageIterator", return_value=mock_iter) - with mock_cls as patched: - from lilbee.vision import _RASTER_DPI, rasterize_pdf + def test_generic_prompt_for_unknown_model(self) -> None: + from lilbee.vision import OCR_PROMPT, resolve_ocr_prompt - list(rasterize_pdf(Path("test.pdf"))) - patched.assert_called_once_with(mock.ANY, dpi=_RASTER_DPI) + assert resolve_ocr_prompt("vendor/qwen-vl") == OCR_PROMPT class TestPngToDataUrl: @@ -139,7 +25,6 @@ def test_encodes_png_bytes(self) -> None: png_bytes = b"\x89PNG\r\n\x1a\n" result = _png_to_data_url(png_bytes) assert result.startswith("data:image/png;base64,") - # Verify round-trip encoded = result.split(",", 1)[1] assert base64.b64decode(encoded) == png_bytes @@ -159,29 +44,3 @@ def test_builds_openai_format(self) -> None: assert content[0]["image_url"]["url"].startswith("data:image/png;base64,") assert content[1]["type"] == "text" assert content[1]["text"] == "describe this" - - -class TestResolveOcrPrompt: - """The OCR prompt is resolved per model: native for specialists, generic fallback otherwise.""" - - def test_deepseek_gets_its_grounding_prompt(self): - from lilbee.vision import resolve_ocr_prompt - - prompt = resolve_ocr_prompt("ggml-org/DeepSeek-OCR-GGUF") - assert prompt == "<|grounding|>Convert the document to markdown." - - def test_glm_ocr_gets_terse_prompt(self): - from lilbee.vision import resolve_ocr_prompt - - assert resolve_ocr_prompt("ggml-org/GLM-OCR-GGUF") == "OCR" - - def test_match_is_case_insensitive_and_works_on_a_gguf_path(self): - from lilbee.vision import resolve_ocr_prompt - - path = "/models/ggml-org/GLM-OCR-GGUF/glm-ocr-Q8_0.gguf" - assert resolve_ocr_prompt(path) == "OCR" - - def test_unknown_model_falls_back_to_generic_prompt(self): - from lilbee.vision import OCR_PROMPT, resolve_ocr_prompt - - assert resolve_ocr_prompt("unsloth/Qwen3-VL-8B-Instruct-GGUF") == OCR_PROMPT diff --git a/tests/test_vision_ocr_backend.py b/tests/test_vision_ocr_backend.py new file mode 100644 index 000000000..95880c750 --- /dev/null +++ b/tests/test_vision_ocr_backend.py @@ -0,0 +1,129 @@ +"""Tests for the lilbee vision OCR backend (kreuzberg custom OCR plugin).""" + +from __future__ import annotations + +import json + +from lilbee.data.ingest.types import MARKDOWN_MIME, OcrBackendName +from lilbee.data.ingest.vision_ocr_backend import ( + VisionOcrBackend, + backend_options_for, + ocr_request, + ocr_requests, +) + + +def _backend(ocr_fn=None, model="vendor/glm-ocr"): + calls: list[tuple] = [] + + def default_fn(image_bytes, model, prompt, *, timeout): + calls.append((image_bytes, model, prompt, timeout)) + return "# extracted" + + be = VisionOcrBackend(ocr_fn=ocr_fn or default_fn, model_ref_fn=lambda: model) + return be, calls + + +def _cfg(*, vlm_prompt=None, backend_options=None): + """The OcrConfig JSON string kreuzberg hands process_image.""" + return json.dumps({"vlm_prompt": vlm_prompt, "backend_options": backend_options}) + + +class TestProtocol: + def test_name_is_enum_value(self): + be, _ = _backend() + assert be.name() == OcrBackendName.LILBEE_VISION == "lilbee-vision" + + def test_backend_type_is_custom(self): + be, _ = _backend() + assert be.backend_type() == "custom" + + def test_supports_all_languages(self): + be, _ = _backend() + assert be.supported_languages() == [] + assert be.supports_language("eng") is True + assert be.supports_language("zho") is True + + def test_version_is_str(self): + be, _ = _backend() + assert isinstance(be.version(), str) and be.version() + + def test_version_falls_back_when_package_missing(self, monkeypatch): + from importlib.metadata import PackageNotFoundError + + from lilbee.data.ingest import vision_ocr_backend as mod + + def _raise(_name): + raise PackageNotFoundError + + monkeypatch.setattr(mod, "version", _raise) + be, _ = _backend() + assert be.version() == "0" + + def test_initialize_shutdown_noop(self): + be, _ = _backend() + assert be.initialize() is None + assert be.shutdown() is None + + +class TestProcessImage: + def test_returns_full_required_schema(self): + be, _ = _backend() + out = be.process_image(b"PNG", _cfg()) + assert out == { + "content": "# extracted", + "mime_type": MARKDOWN_MIME, + "metadata": {}, + "tables": [], + "chunks": [], + "images": [], + } + + def test_passes_model_and_resolved_prompt(self): + be, calls = _backend(model="vendor/glm-ocr") + be.process_image(b"PNG", _cfg()) + _, model, prompt, _ = calls[0] + assert model == "vendor/glm-ocr" + # glm-ocr has a native prompt; resolve_ocr_prompt picks it, not the generic one. + assert prompt == "OCR" + + def test_vlm_prompt_overrides_resolved(self): + be, calls = _backend() + be.process_image(b"PNG", _cfg(vlm_prompt="custom prompt")) + assert calls[0][2] == "custom prompt" + + def test_request_context_supplies_timeout_and_fires_progress(self): + ticks: list[int] = [] + be, calls = _backend() + with ocr_request(on_page=lambda: ticks.append(1), timeout=12.5) as token: + be.process_image(b"PNG", _cfg(backend_options=backend_options_for(token))) + assert calls[0][3] == 12.5 + assert ticks == [1] + + def test_no_context_uses_zero_timeout_and_no_tick(self): + be, calls = _backend() + be.process_image(b"PNG", _cfg(backend_options=backend_options_for("unknown-token"))) + assert calls[0][3] == 0.0 + + def test_malformed_backend_options_ignored(self): + be, calls = _backend() + be.process_image(b"PNG", _cfg(backend_options="not-json")) + assert calls[0][3] == 0.0 + + +class TestRegistry: + def test_token_registered_within_scope_and_cleaned_after(self): + with ocr_request(timeout=3.0) as token: + ctx = ocr_requests.get(token) + assert ctx is not None and ctx.timeout == 3.0 + assert ocr_requests.get(token) is None + + def test_get_none_token_returns_none(self): + assert ocr_requests.get(None) is None + + def test_backend_options_round_trip(self): + token = "abc123" + be, calls = _backend() + be.process_image(b"PNG", _cfg(backend_options=backend_options_for(token))) + # token not registered -> no context, zero timeout + assert calls[0][3] == 0.0 diff --git a/uv.lock b/uv.lock index 463d73326..e8db7d860 100644 --- a/uv.lock +++ b/uv.lock @@ -1454,14 +1454,10 @@ wheels = [ [[package]] name = "kreuzberg" -version = "4.9.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/67/d4/6559b625ab7606ffebf9b025745bc202296df1a788573b406785b8214366/kreuzberg-4.9.2.tar.gz", hash = "sha256:59a99e14e2156530ef0b1344be53baef50bde87f0908099ac6127a88e0c9bbc1", size = 2437544 } +version = "5.0.0rc32" +source = { path = "../../../../kreuzberg/.worktrees/rc32/dist/kreuzberg-5.0.0rc32-cp310-abi3-macosx_11_0_arm64.whl" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/a1/acb48088d55b2387c2cfc4d24e92248fc0ef0657f3acb20b80b391c97a8d/kreuzberg-4.9.2-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:5113ff1b9878fa92eb00c9cbb8b1d06600a770477e76f0045d20788a50c7ac7e", size = 28753304 }, - { url = "https://files.pythonhosted.org/packages/f3/40/4d58dcdf4cc14180183afc6ea2fa1b504b64f5154b8fea3d16930ff9cc65/kreuzberg-4.9.2-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:953fd353023585873f33f93d1e03f4e4bb747aef0c86320c450db3f8f10acef3", size = 33258395 }, - { url = "https://files.pythonhosted.org/packages/ec/bc/c12c0777b83069f1228e5a4bb341e69aa03ce5c6fe9f7b65936931b8c94a/kreuzberg-4.9.2-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c4c6d300d03b0ea8936e1b3faedf22d559cb790bb364417121df22bb81f890a6", size = 35500862 }, - { url = "https://files.pythonhosted.org/packages/48/b3/fb0c529d4ff2935dd17bb52f334d95425e10c2b1786929112302f82da8a7/kreuzberg-4.9.2-cp310-abi3-win_amd64.whl", hash = "sha256:0a4b905df6534e88eeaa17dc27cfbc3dd49283085bedc658a447d4c9499c8f6b", size = 33066471 }, + { filename = "kreuzberg-5.0.0rc32-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:79477dcd3e77e8401baa82620cb51ae500f3c553c3a251e8f204ca64cc76da9d" }, ] [[package]] @@ -1673,7 +1669,7 @@ requires-dist = [ { name = "httpx" }, { name = "huggingface-hub", specifier = ">=1.11.0" }, { name = "jinja2", specifier = ">=2.11.3" }, - { name = "kreuzberg", specifier = ">=4.9.1,<5" }, + { name = "kreuzberg", path = "../../../../kreuzberg/.worktrees/rc32/dist/kreuzberg-5.0.0rc32-cp310-abi3-macosx_11_0_arm64.whl" }, { name = "lancedb" }, { name = "lilbee", extras = ["crawler", "litellm", "graph"], marker = "extra == 'release'" }, { name = "lilbee-engine", directory = "packaging/engine-wheel" }, From a5bee1eeb9aecb74eb2651e575ffec769ecb73ad Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Wed, 24 Jun 2026 21:25:24 -0400 Subject: [PATCH 02/77] feat: support all kreuzberg formats + configurable OCR language Derive supported formats from kreuzberg.list_supported_formats() (PDFs and images grouped, other formats keyed by extension) instead of a hand-maintained 38-entry map, so lilbee covers all ~108 formats kreuzberg extracts; source code still routes to the tree-sitter chunker. Add cfg.ocr_language (writable; env LILBEE_OCR_LANGUAGE), fixing image+tesseract OCR that kreuzberg 5.x errors on with an empty language list. Drop the dead max_concurrent_extractions knob and the OcrConfig str|dict shim. Regroup ingest types; add the AGENTS.md canonical-module-layout rule and a diff-scoped scatter trigger. Update the README format table; add coverage tests. --- AGENTS.md | 3 +- README.md | 16 ++--- src/lilbee/app/settings_map.py | 6 ++ src/lilbee/core/config/model.py | 16 +++++ src/lilbee/data/ingest/discovery.py | 39 ++++++++++++- src/lilbee/data/ingest/extract.py | 14 ++--- src/lilbee/data/ingest/types.py | 61 +++----------------- src/lilbee/data/ingest/vision_ocr_backend.py | 23 +++----- tests/test_config.py | 28 +++++++++ tests/test_ingest.py | 46 +++++++++------ tests/test_vision_ocr_backend.py | 14 +++++ 11 files changed, 162 insertions(+), 104 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ac8d1fd17..5552572d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -139,7 +139,7 @@ CLI also accepts `--model` / `-m` for chat model, `--data-dir` / `-d`, `--ocr-ti - Type checking: `mypy` with strict settings - **No em dashes (—)** — use periods or commas instead - **No divider comments** (`# ------`) for grouping — use modules or classes instead -- **Don't scatter functions between constants.** Within a module (or a topical block in one), keep the module-level constants contiguous and the functions/classes contiguous. A `def` wedged into the middle of a run of `FOO = ...` / `BAR = ...` lines breaks the eye's scan. A helper that formats or validates a small group of constants may follow that group directly (the constants, then their one helper), but don't interleave: `A = ...`, `def f(): ...`, `B = ...`, `def g(): ...`, `C = ...` is wrong; group it as `A = ...`, `B = ...`, `C = ...`, `def f(): ...`, `def g(): ...`. Order by concern (all the "skip" constants, then the skip helpers; all the "doc table" constants, then the doc-table dataclass), not by when each was added. +- **Canonical module layout — group by kind, don't scatter.** A module reads top-to-bottom in this order: module docstring → `from __future__` → imports → module-level constants & data tables → enums → exception classes → data containers (`dataclass` / `TypedDict` / `NamedTuple` / `BaseModel`) → classes → functions. Within each band, order by concern, not by when each symbol was added. The reader should never have to scan past a `class` or `def` to find the next constant, nor past a constant to find the next class. A constant assignment (`FOO = ...`, `BAR: T = ...`) appearing AFTER the first `def`/`class` in a module is the scatter smell — hoist it into the constants band at the top. A `def` wedged into a run of `FOO = ...` / `BAR = ...` lines is the same smell from the other side. The one allowed exception: a single small helper that exists only to build or validate one adjacent group of constants may sit directly after that group (the constants, then their one helper) — but never interleaved (`A = ...`, `def f(): ...`, `B = ...` is always wrong). When in doubt, constants and data tables go at the top, contiguous; everything callable goes below, contiguous. - **Annotate `isinstance` guards** with a comment explaining why the check is needed (e.g. untyped frontmatter, untyped SDK response). NEVER use `isinstance(self.app, LilbeeApp)` style host-narrowing in production: if a screen / widget needs LilbeeApp-specific access, declare `app: LilbeeApp # type: ignore[assignment]` at the class scope (mypy narrows; runtime is unchanged). Tests host via `tests/_lilbee_app_test_host.LilbeeAppHost` (a LilbeeApp subclass with `_test_skip_auto_init = True`), never `App[None]` / `class _PlainApp(App)`. - Use literal unicode chars (`★`) not escapes (`\u2605`); extract to named constants - **No filterwarnings** without explicit user approval; fix warnings at the source @@ -389,6 +389,7 @@ work; the greps below remain the full set a reviewer still runs by eye. - `grep -rn "App\[None\]\|class _PlainApp\b\|class _BareApp\b" tests/` — test host that bypasses LilbeeApp. Migrate to LilbeeAppHost. - `grep -cE "isinstance\(.*widget, (Input|Checkbox|Select|TextArea)\)" src/` followed by "if more than one site" — type-keyed widget routing. Replace with a `dict[type, Callable]`. - `grep -rnE "^\s*global \w+" src/` — module-level mutable globals. Encapsulate on a class. +- `for f in $(git diff --name-only main -- 'src/**/*.py'); do awk 'NR>1 && /^[A-Z_]+ *[:=]/ && seen {print FILENAME": "$0} /^(def |class )/{seen=1}' "$f"; done` — module-level constant assigned AFTER the first `def`/`class` (the scatter smell). Hoist it into the constants band at the top. See the canonical-module-layout rule in Code Style. - `grep -rnE "def \w+\(.*\) -> .*:\s*$" src/ | xargs -I{} awk 'def_lines>20'` (proxy: scan modules >700 LOC) — functions over ~20 lines need a split into named sub-helpers. - `grep -rn "from lilbee\.\(modelhub\|providers\|data\|retrieval\|wiki\|crawler\)" src/lilbee/catalog src/lilbee/core` — upward imports from foundation layers. catalog and core never import from anything above them. - `grep -rn "from lilbee\.\(retrieval\|wiki\|crawler\|cli\|server\)" src/lilbee/data src/lilbee/modelhub src/lilbee/providers` — Layer-2 modules importing Layer-3+. Invert via callback or move the helper down. diff --git a/README.md b/README.md index d8e0b4b54..0c8cb64db 100644 --- a/README.md +++ b/README.md @@ -358,20 +358,22 @@ Pull a chat and embedding model first; all recipes pin the server to `127.0.0.1: ## Supported formats -Text extraction powered by [Kreuzberg], code chunking by [tree-sitter]. Structured formats (XML, JSON, CSV) get embedding-friendly preprocessing. This list is not exhaustive; Kreuzberg supports additional formats beyond what's listed here. +Document extraction powered by [Kreuzberg], code chunking by [tree-sitter]. lilbee handles every format Kreuzberg can extract (100+) and tracks its list directly, so support grows as Kreuzberg adds formats. The table below covers the common ones. | Format | Extensions | Requires | | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | PDF | `.pdf` | none | | Scanned PDF | `.pdf` (no extractable text) | [Tesseract](https://github.com/tesseract-ocr/tesseract) (auto, plain text), or a GGUF vision model via the native mtmd backend (recommended, preserves tables, headings, and layout as markdown) | -| Office | `.docx`, `.xlsx`, `.pptx` | none | -| eBook | `.epub` | none | -| Images (OCR) | `.png`, `.jpg`, `.jpeg`, `.tiff`, `.bmp`, `.webp` | [Tesseract](https://github.com/tesseract-ocr/tesseract) | -| Data | `.csv`, `.tsv` | none | -| Structured | `.xml`, `.json`, `.jsonl`, `.yaml`, `.yml` | none | +| Office | `.docx`, `.xlsx`, `.pptx`, `.doc`, `.xls`, `.ppt`, `.odt`, `.ods`, `.rtf` | none | +| eBook | `.epub`, `.fb2` | none | +| Images (OCR) | `.png`, `.jpg`, `.jpeg`, `.tiff`, `.bmp`, `.webp`, `.gif`, `.heic`, `.avif`, `.jp2`/`.j2k`/`.jpx` (JPEG-2000) | [Tesseract](https://github.com/tesseract-ocr/tesseract) or a vision model | +| Text/markup | `.md`, `.txt`, `.html`, `.rst`, `.org`, `.tex`, `.typst` | none | +| Data | `.csv`, `.tsv`, `.json`, `.jsonl`, `.xml`, `.yaml`, `.toml` | none | +| Email | `.eml`, `.msg` | none | +| Archives | `.zip`, `.tar`, `.gz`, `.7z`, `.pst` (combined contents extracted) | none | | Code | `.py`, `.js`, `.ts`, `.go`, `.rs`, `.java` and [150+ more](https://github.com/Goldziher/tree-sitter-language-pack) via tree-sitter (AST-aware chunking) | none | -See the [usage guide](docs/usage.md#ocr) for OCR setup and [model benchmarks](docs/benchmarks/vision-ocr.md). +Plus notebooks, bibliographies, iWork, and audio/video, among others. See the [usage guide](docs/usage.md#ocr) for OCR setup and [model benchmarks](docs/benchmarks/vision-ocr.md). ## Experimental diff --git a/src/lilbee/app/settings_map.py b/src/lilbee/app/settings_map.py index 96789b95f..83348c50e 100644 --- a/src/lilbee/app/settings_map.py +++ b/src/lilbee/app/settings_map.py @@ -721,6 +721,12 @@ def get_default(key: str) -> object: group=SettingGroup.INGEST, help_text="Per-page Tesseract timeout in seconds (used when no vision model is set)", ), + "ocr_language": SettingDef( + list, + nullable=False, + group=SettingGroup.INGEST, + help_text="Tesseract OCR language codes when no vision model is set (e.g. eng, eng+deu)", + ), "worker_pool_eager_start": SettingDef( bool, nullable=False, diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index f9508d303..6d15cd0e4 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -132,6 +132,10 @@ class Config(BaseSettings): # Tesseract fallback wall-clock timeout per file, seconds. 0 = no cap. tesseract_timeout: float = ConfigField(default=60.0, ge=0.0, writable=True) + # Tesseract OCR language codes for the scanned-document fallback (used when no + # vision model is set), e.g. ["eng"] or ["eng", "deu"]. Set via env as + # LILBEE_OCR_LANGUAGE="eng+deu". kreuzberg requires a non-empty list. + ocr_language: list[str] = ConfigField(default_factory=lambda: ["eng"], writable=True) semantic_chunking: bool = ConfigField(default=False, writable=True) topic_threshold: float = ConfigField(default=0.75, ge=0.0, le=1.0, writable=True) server_host: str = "127.0.0.1" @@ -678,6 +682,18 @@ def _parse_enable_ocr(cls, v: Any) -> bool | None: pass # fall through to bool() coercion below for unrecognised strings return bool(v) + @field_validator("ocr_language", mode="before") + @classmethod + def _parse_ocr_language(cls, v: Any) -> list[str]: + """Accept a list or a comma/plus-separated string; never return empty. + + Tesseract uses ``+`` to join languages and kreuzberg errors on an empty + list, so blank input falls back to English. + """ + items = v.replace("+", ",").split(",") if isinstance(v, str) else (v or []) + langs = [s.strip() for s in items if isinstance(s, str) and s.strip()] + return langs or ["eng"] + @field_validator("flash_attention", mode="before") @classmethod def _parse_flash_attention(cls, v: Any) -> bool | None: diff --git a/src/lilbee/data/ingest/discovery.py b/src/lilbee/data/ingest/discovery.py index aba2eabe4..c638f302c 100644 --- a/src/lilbee/data/ingest/discovery.py +++ b/src/lilbee/data/ingest/discovery.py @@ -5,16 +5,45 @@ import hashlib import logging import os +from functools import cache from pathlib import Path from lilbee.core.config import cfg from lilbee.core.security import validate_path_within from lilbee.core.system import is_ignored_dir from lilbee.data.code_chunker import is_code_file -from lilbee.data.ingest.types import DOCUMENT_EXTENSION_MAP +from lilbee.data.ingest.types import IMAGE_CONTENT_TYPE, PDF_CONTENT_TYPE log = logging.getLogger(__name__) +_PDF_MIME = "application/pdf" + + +def _content_type_for(ext: str, mime: str) -> str: + """content_type for a kreuzberg format: PDFs and images grouped, others keyed by extension.""" + if mime == _PDF_MIME: + return PDF_CONTENT_TYPE + if mime.startswith("image/"): + return IMAGE_CONTENT_TYPE + return ext.lstrip(".") + + +@cache +def supported_extension_map() -> dict[str, str]: + """Extension -> content_type for every format kreuzberg can extract. + + Built from ``kreuzberg.list_supported_formats()`` so lilbee covers the full set + without a hand-maintained list. Source-code files are routed separately (their + extensions are absent here), so ``classify_file`` falls through to the code path. + """ + from kreuzberg import list_supported_formats + + out: dict[str, str] = {} + for fmt in list_supported_formats(): + ext = (fmt.extension if fmt.extension.startswith(".") else f".{fmt.extension}").lower() + out[ext] = _content_type_for(ext, fmt.mime_type) + return out + def file_hash(path: Path) -> str: """Compute SHA-256 hex digest of a file.""" @@ -31,8 +60,12 @@ def _relative_name(path: Path) -> str: def classify_file(path: Path) -> str | None: - """Classify file by extension. Returns content_type or None if unsupported.""" - doc_type = DOCUMENT_EXTENSION_MAP.get(path.suffix.lower()) + """Classify a file by extension: a kreuzberg content_type, "code", or None. + + kreuzberg-extractable formats win; source code (not in kreuzberg's set) routes + to the code chunker; anything else is unsupported. + """ + doc_type = supported_extension_map().get(path.suffix.lower()) if doc_type is not None: return doc_type if is_code_file(path): diff --git a/src/lilbee/data/ingest/extract.py b/src/lilbee/data/ingest/extract.py index 37ec6da9d..0839e2f26 100644 --- a/src/lilbee/data/ingest/extract.py +++ b/src/lilbee/data/ingest/extract.py @@ -24,7 +24,6 @@ ) from lilbee.data.ingest.vision_ocr_backend import backend_options_for, ocr_request from lilbee.data.store import ChunkType, PageTextRecord -from lilbee.runtime.cpu import cpu_quota from lilbee.runtime.progress import ( DetailedProgressCallback, EventType, @@ -114,30 +113,31 @@ def _ocr_config(ocr_token: str | None) -> OcrConfig: if cfg.vision_model: options = backend_options_for(ocr_token) if ocr_token else None return OcrConfig(backend=OcrBackendName.LILBEE_VISION, backend_options=options) - return OcrConfig(backend=OcrBackendName.TESSERACT) + # kreuzberg requires a non-empty language list (4.x defaulted to English; + # 5.x errors on an empty one). cfg.ocr_language is validated non-empty. + return OcrConfig(backend=OcrBackendName.TESSERACT, language=list(cfg.ocr_language)) def extraction_config(mode: ExtractMode, *, ocr_token: str | None = None) -> ExtractionConfig: """Build ExtractionConfig for the given extraction mode.""" from kreuzberg import ExtractionConfig, PageConfig + # Files are extracted one per call, so kreuzberg parallelizes OCR across the + # pages of each document internally; cross-file concurrency is the pipeline's + # semaphore. (max_concurrent_extractions only bounds multi-file batch calls, + # which lilbee never makes, so it is intentionally not set here.) chunking = build_chunking_config() ocr = _ocr_config(ocr_token) - # Bound batch extraction to the CPU budget so kreuzberg and the pipeline - # semaphore stop competing for cores. - max_concurrent = cpu_quota() if mode is ExtractMode.PAGINATED: return ExtractionConfig( chunking=chunking, pages=PageConfig(extract_pages=True, insert_page_markers=False), ocr=ocr, - max_concurrent_extractions=max_concurrent, ) return ExtractionConfig( chunking=chunking, output_format=MARKDOWN_OUTPUT, ocr=ocr, - max_concurrent_extractions=max_concurrent, ) diff --git a/src/lilbee/data/ingest/types.py b/src/lilbee/data/ingest/types.py index 2e3ef5ae6..96c923906 100644 --- a/src/lilbee/data/ingest/types.py +++ b/src/lilbee/data/ingest/types.py @@ -17,6 +17,14 @@ SourceStatBackfill, ) +# PDF and image content types route to paginated extraction; every other format +# routes to markdown extraction. content_type is derived per-file in +# discovery.classify_file (PDFs and images grouped; others keyed by extension). +PDF_CONTENT_TYPE = "pdf" +IMAGE_CONTENT_TYPE = "image" +MARKDOWN_OUTPUT = "markdown" +MARKDOWN_MIME = "text/markdown" + class FileToProcess(NamedTuple): """A file queued for ingestion with its metadata.""" @@ -39,12 +47,6 @@ class FileChangePlan(NamedTuple): stat_backfills: list[SourceStatBackfill] -PDF_CONTENT_TYPE = "pdf" -IMAGE_CONTENT_TYPE = "image" -MARKDOWN_OUTPUT = "markdown" -MARKDOWN_MIME = "text/markdown" - - class OcrBackendName(StrEnum): """OCR backends lilbee selects in OcrConfig: kreuzberg's tesseract or lilbee's vision plugin.""" @@ -137,50 +139,3 @@ class _IngestResult: page_texts: list[PageTextRecord] | None = None stat: SourceStat | None = None concept_records: ConceptRecords | None = None - - -# Extension → content_type string for document formats handled by kreuzberg. -# Container formats (.zip/.tar/.7z/.pst) are deliberately absent: kreuzberg can -# extract them, but one file fans out to many inner documents, which the -# source/citation model can't express yet. -DOCUMENT_EXTENSION_MAP: dict[str, str] = { - **{ext: "text" for ext in (".md", ".txt", ".html", ".htm", ".rst", ".yaml", ".yml")}, - ".pdf": PDF_CONTENT_TYPE, - **{ - ext: ext.lstrip(".") - for ext in ( - ".docx", - ".xlsx", - ".pptx", - ".doc", - ".xls", - ".ppt", - ".rtf", - ".odt", - ".ods", - ".eml", - ".msg", - ) - }, - ".epub": "epub", - **{ - ext: IMAGE_CONTENT_TYPE - for ext in ( - ".png", - ".jpg", - ".jpeg", - ".tiff", - ".tif", - ".bmp", - ".webp", - ".gif", - ".jp2", - ".j2k", - ".j2c", - ".jpx", - ) - }, - **{ext: "data" for ext in (".csv", ".tsv", ".dbf")}, - ".xml": "xml", - **{ext: "json" for ext in (".json", ".jsonl")}, -} diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py index 14ec40a2d..781c81547 100644 --- a/src/lilbee/data/ingest/vision_ocr_backend.py +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -74,19 +74,12 @@ def backend_options_for(token: str) -> str: return json.dumps({_REQUEST_TOKEN_KEY: token}) -def _config_to_dict(config: str | dict[str, Any]) -> dict[str, Any]: - """Normalize the OcrConfig kreuzberg hands process_image into a dict. - - kreuzberg serializes the OcrConfig to a JSON string for the callback (rc.35); - an already-parsed dict is accepted as-is for forward compatibility. - """ - if isinstance(config, str): - try: - raw: object = json.loads(config) - except (ValueError, TypeError): - return {} - else: - raw = config +def _config_to_dict(config: str) -> dict[str, Any]: + """Parse the OcrConfig JSON string kreuzberg passes to process_image into a dict.""" + try: + raw: object = json.loads(config) + except (ValueError, TypeError): + return {} return cast("dict[str, Any]", raw) if isinstance(raw, dict) else {} @@ -155,9 +148,9 @@ def shutdown(self) -> None: ... def backend_type(self) -> str: return "custom" - def process_image(self, image_bytes: bytes, config: str | dict[str, Any]) -> dict[str, Any]: + def process_image(self, image_bytes: bytes, config: str) -> dict[str, Any]: # kreuzberg serializes the OcrConfig to a JSON string for the callback; - # normalize before reading fields. + # parse before reading fields. view = _OcrConfigView(_config_to_dict(config)) model = self._model_ref_fn() prompt = view.vlm_prompt or resolve_ocr_prompt(model) diff --git a/tests/test_config.py b/tests/test_config.py index 3103661e2..254b3beca 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -145,6 +145,34 @@ def test_chat_model_override_native_hf_ref(self): c = Config() assert c.chat_model == ref + +class TestOcrLanguage: + def test_defaults_to_english(self, tmp_path): + with mock.patch.dict(os.environ, _clean_env(tmp_path), clear=True): + assert Config().ocr_language == ["eng"] + + def test_env_plus_separated(self, tmp_path): + env = _clean_env(tmp_path) | {"LILBEE_OCR_LANGUAGE": "eng+deu"} + with mock.patch.dict(os.environ, env, clear=True): + assert Config().ocr_language == ["eng", "deu"] + + def test_env_comma_separated(self, tmp_path): + env = _clean_env(tmp_path) | {"LILBEE_OCR_LANGUAGE": "deu, fra"} + with mock.patch.dict(os.environ, env, clear=True): + assert Config().ocr_language == ["deu", "fra"] + + def test_direct_list(self): + assert Config(ocr_language=["spa"]).ocr_language == ["spa"] + + def test_empty_list_falls_back_to_english(self): + assert Config(ocr_language=[]).ocr_language == ["eng"] + + def test_blank_string_falls_back_to_english(self): + assert Config(ocr_language="").ocr_language == ["eng"] + + def test_blank_entries_are_dropped(self): + assert Config(ocr_language=["", "eng", " "]).ocr_language == ["eng"] + def test_chat_mode_defaults_to_search_when_none_or_empty(self): """The validator coerces None / "" to 'search' so old configs round-trip.""" from lilbee.core.config.model import Config as ConfigCls diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 3e66f1f75..af0aed4ab 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -282,7 +282,7 @@ async def test_unchanged_file_skipped(self, mock_extract_file, isolated_env): assert result.added == [] async def test_unsupported_extension_skipped(self, mock_extract_file, isolated_env): - (isolated_env / "data.zip").write_bytes(b"binary data") + (isolated_env / "data.exe").write_bytes(b"binary data") from lilbee.data.ingest import sync assert (await sync()).added == [] @@ -1423,14 +1423,14 @@ class TestClassifyFile: "filename, expected", [ ("doc.pdf", "pdf"), - ("f.md", "text"), - ("f.txt", "text"), - ("f.html", "text"), - ("f.rst", "text"), + ("f.md", "md"), + ("f.txt", "txt"), + ("f.html", "html"), + ("f.rst", "rst"), ("f.py", "code"), ("f.js", "code"), ("f.go", "code"), - ("f.zip", None), + ("f.zip", "zip"), ("f.exe", None), ], ) @@ -1846,8 +1846,8 @@ class TestClassifyNewFormats: ("scan.tif", "image"), ("img.bmp", "image"), ("img.webp", "image"), - ("data.csv", "data"), - ("data.tsv", "data"), + ("data.csv", "csv"), + ("data.tsv", "tsv"), ], ) def test_classify(self, filename, expected): @@ -1948,6 +1948,17 @@ def test_paginated_has_tesseract_ocr_backend(self): assert config.get("ocr") is not None # No vision model configured -> kreuzberg's tesseract backend OCRs scanned pages. assert config["ocr"].backend == "tesseract" + # kreuzberg 5.x errors on image OCR with an empty language list; lilbee must + # set an explicit default to preserve 4.x behavior (regression guard). + assert config["ocr"].language == ["eng"] + + def test_tesseract_ocr_language_from_config(self, monkeypatch): + from lilbee.core.config import cfg + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.setattr(cfg, "ocr_language", ["deu", "fra"]) + config = extraction_config(ExtractMode.PAGINATED) + assert config["ocr"].language == ["deu", "fra"] @pytest.mark.parametrize( "content_type, expected_mode_name", @@ -1991,7 +2002,7 @@ class TestClassifyKreuzbergParityFormats: ("scan.j2k", "image"), ("scan.j2c", "image"), ("scan.jpx", "image"), - ("page.htm", "text"), + ("page.htm", "htm"), ("memo.doc", "doc"), ("deck.ppt", "ppt"), ("ledger.xls", "xls"), @@ -2000,11 +2011,10 @@ class TestClassifyKreuzbergParityFormats: ("ledger.ods", "ods"), ("mail.eml", "eml"), ("mail.msg", "msg"), - ("table.dbf", "data"), - # Containers stay unsupported on purpose: one file fans out to many - # inner documents, which the source/citation model can't express yet. - ("archive.pst", None), - ("archive.7z", None), + ("table.dbf", "dbf"), + # Containers are supported too (kreuzberg extracts their combined text). + ("archive.pst", "pst"), + ("archive.7z", "7z"), ], ) def test_classify(self, filename, expected): @@ -2019,10 +2029,10 @@ class TestClassifyStructuredFormats: [ ("data.xml", "xml"), ("data.json", "json"), - ("data.jsonl", "json"), - ("config.yaml", "text"), - ("config.yml", "text"), - ("data.csv", "data"), + ("data.jsonl", "jsonl"), + ("config.yaml", "yaml"), + ("config.yml", "yml"), + ("data.csv", "csv"), ], ) def test_classify(self, filename, expected): diff --git a/tests/test_vision_ocr_backend.py b/tests/test_vision_ocr_backend.py index 95880c750..b39df7a08 100644 --- a/tests/test_vision_ocr_backend.py +++ b/tests/test_vision_ocr_backend.py @@ -110,6 +110,20 @@ def test_malformed_backend_options_ignored(self): be.process_image(b"PNG", _cfg(backend_options="not-json")) assert calls[0][3] == 0.0 + def test_malformed_config_string_falls_back_to_defaults(self): + # A non-JSON config string yields an empty view: resolved prompt, zero timeout. + be, calls = _backend(model="vendor/glm-ocr") + be.process_image(b"PNG", "}{ not json") + _, _, prompt, timeout = calls[0] + assert prompt == "OCR" + assert timeout == 0.0 + + def test_non_object_json_config_falls_back_to_defaults(self): + # Valid JSON that isn't an object (e.g. a bare number) is treated as empty. + be, calls = _backend(model="vendor/glm-ocr") + be.process_image(b"PNG", "123") + assert calls[0][2] == "OCR" + class TestRegistry: def test_token_registered_within_scope_and_cleaned_after(self): From 7dd5df6a61c80e8ab463c9311feff1092f41ce59 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:36:03 -0400 Subject: [PATCH 03/77] feat: migrate kreuzberg -> xberg 1.0.0rc1 The dependency was renamed kreuzberg -> xberg (now v1.0.0rc1). Rename every reference across code, tests, docs, README, the marketing site, AGENTS.md and the build script; the internal type module is now xberg._xberg. Build the semantic chunking embedding config with the real EmbeddingModelType.preset() constructor (alef #147) instead of the bare-string shorthand. xberg's .pyi omits those per-variant constructors (alef-m07 / kreuzberg-7xq), so the one call carries a narrow type-ignore. API is otherwise compatible; the oracle differential shows identical extraction/OCR/chunking vs the kreuzberg-5.x baseline. The local wheel source is a placeholder until xberg publishes to an index. --- .gitignore | 2 +- AGENTS.md | 2 +- README.md | 8 +- docs/architecture.md | 8 +- docs/benchmarks/vision-ocr.md | 2 +- docs/usage.md | 2 +- pyproject.toml | 6 +- site/index.html | 2 +- src/lilbee/app/services.py | 8 +- src/lilbee/app/settings.py | 2 +- src/lilbee/cli/tui/__init__.py | 4 +- src/lilbee/core/config/model.py | 4 +- src/lilbee/data/chunk.py | 14 +-- src/lilbee/data/ingest/discovery.py | 12 +-- src/lilbee/data/ingest/extract.py | 30 +++---- src/lilbee/data/ingest/types.py | 2 +- src/lilbee/data/ingest/vision_ocr_backend.py | 22 ++--- src/lilbee/providers/fleet/provider.py | 4 +- src/lilbee/runtime/progress/types.py | 2 +- src/lilbee/skills/lilbee_mcp/SKILL.md | 2 +- src/lilbee/vision.py | 2 +- tests/integration/test_pdf_integration.py | 10 +-- tests/server/test_handlers.py | 24 +++--- tests/test_chunker.py | 13 +-- tests/test_dynamic_settings_eviction.py | 8 +- tests/test_formats.py | 36 ++++---- tests/test_ingest.py | 90 ++++++++++---------- tests/test_services.py | 20 ++--- tests/test_vision_ocr_backend.py | 4 +- tools/wheel-build/build_lilbee_binary.sh | 2 +- uv.lock | 20 ++--- 31 files changed, 187 insertions(+), 180 deletions(-) diff --git a/.gitignore b/.gitignore index 3437647f9..e33333a51 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,7 @@ CLAUDE.md docs/plans/ demo.cast docs/superpowers/ -.kreuzberg/ +.xberg/ .claude/ .dolt/ .doltcfg/ diff --git a/AGENTS.md b/AGENTS.md index 5552572d0..c28b6b87f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -189,7 +189,7 @@ Default: **every import lives at module top**, ordered stdlib, third-party, loca **Permitted reasons for a function-local import:** 1. **Circular import.** Module A's top-level already imports module B, and module B needs a symbol from A. Put A's import of B inside the one function in B that needs it, with a comment `# circular: B -> A via `. -2. **Heavy third-party lib.** Module-top import takes **>50 ms measured by `python -X importtime`** or loads native libraries. Known-heavy libs in this project: `litellm` (provider SDK fanout), `lancedb` (arrow + datafusion), `kreuzberg` (OCR stack), `gguf` (numpy), `sentence_transformers`, `spacy`, `crawl4ai`, `textual` when imported outside the TUI screen modules. (The local inference engine is the out-of-process `llama-server`, reached over HTTP, so it adds no heavy Python import.) Use importtime before declaring a lib heavy. +2. **Heavy third-party lib.** Module-top import takes **>50 ms measured by `python -X importtime`** or loads native libraries. Known-heavy libs in this project: `litellm` (provider SDK fanout), `lancedb` (arrow + datafusion), `xberg` (OCR stack), `gguf` (numpy), `sentence_transformers`, `spacy`, `crawl4ai`, `textual` when imported outside the TUI screen modules. (The local inference engine is the out-of-process `llama-server`, reached over HTTP, so it adds no heavy Python import.) Use importtime before declaring a lib heavy. 3. **CLI startup path.** The import lives inside a Typer command body so `lilbee --help` stays fast. Treat this as a sub-case of (2): the CLI loader stays lean so unused subcommands don't pay for their dependencies. **Never lazy-import the following:** diff --git a/README.md b/README.md index 0c8cb64db..848b3dbf7 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ Or crawl a whole site, not just one page. With recursive crawling on, lilbee fol lilbee splits indexing by what's being read: -- **Prose and structured documents** (PDFs, Office files, ebooks, HTML, 90+ formats) go through [Kreuzberg] with heading-aware chunking, so each chunk keeps its section context. +- **Prose and structured documents** (PDFs, Office files, ebooks, HTML, 90+ formats) go through [Xberg] with heading-aware chunking, so each chunk keeps its section context. - **Code** goes through [tree-sitter]'s AST-aware splitter across [150+ languages](https://github.com/Goldziher/tree-sitter-language-pack), so chunks map to functions, classes, and modules instead of arbitrary line ranges. - **Scanned PDFs and photos** go through OCR: Tesseract for plain text, or a local / remote vision model that keeps tables and layout as markdown. @@ -358,7 +358,7 @@ Pull a chat and embedding model first; all recipes pin the server to `127.0.0.1: ## Supported formats -Document extraction powered by [Kreuzberg], code chunking by [tree-sitter]. lilbee handles every format Kreuzberg can extract (100+) and tracks its list directly, so support grows as Kreuzberg adds formats. The table below covers the common ones. +Document extraction powered by [Xberg], code chunking by [tree-sitter]. lilbee handles every format Xberg can extract (100+) and tracks its list directly, so support grows as Xberg adds formats. The table below covers the common ones. | Format | Extensions | Requires | | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | @@ -402,7 +402,7 @@ See the [Semantic chunking section of the usage guide](docs/usage.md#semantic-ch lilbee stands on a stack of established open-source projects, all bundled into one install: -- [Kreuzberg] parses 90+ document formats with heading-aware chunking. +- [Xberg] parses 90+ document formats with heading-aware chunking. - [llama.cpp] is the local model runtime: lilbee bundles its `llama-server` and starts it for you, so every chat, embedding, vision, and reranker call goes through it. [llama-swap] keeps a server per role resident together behind one endpoint, and [gguf-parser] estimates each model's memory footprint so lilbee loads what fits. Without llama.cpp there is no lilbee. - [Hugging Face Hub] (via [huggingface_hub]) hosts the model catalog and handles every download. Search, browse, and pull all route through it. - [LanceDB] is the embedded vector store. @@ -423,7 +423,7 @@ lilbee is built and maintained by one person. If it is useful to you, you can ch Elastic License 2.0 (ELv2). See [LICENSE](LICENSE). -[Kreuzberg]: https://github.com/kreuzberg-dev/kreuzberg +[Xberg]: https://github.com/xberg-io/xberg [LanceDB]: https://lancedb.com [llama.cpp]: https://github.com/ggml-org/llama.cpp [llama-swap]: https://github.com/mostlygeek/llama-swap diff --git a/docs/architecture.md b/docs/architecture.md index c4a55c0fa..abb32f3b4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -73,13 +73,13 @@ flowchart LR Documents are chunked, embedded, and stored as vectors for later retrieval. - **File discovery.** Recursive walk of `documents/` with SHA-256 hash-based change detection so only modified files are re-indexed. Each source row also stores the file's size and mtime, so an unchanged file is skipped on a stat alone; the hash runs only when the stat pair drifts. Stores created before these columns re-hash once and backfill. -- **Markdown.** Heading-aware chunking via kreuzberg's `chunker_type="markdown"` with `prepend_heading_context=True`. Splits at heading boundaries and prepends the full hierarchy path (e.g., `# Setup > ## Install`) so each chunk's section context travels with it. Inspired by Anthropic's Contextual Retrieval (2024), which showed adding context to chunks reduces retrieval failures by 49%. +- **Markdown.** Heading-aware chunking via xberg's `chunker_type="markdown"` with `prepend_heading_context=True`. Splits at heading boundaries and prepends the full hierarchy path (e.g., `# Setup > ## Install`) so each chunk's section context travels with it. Inspired by Anthropic's Contextual Retrieval (2024), which showed adding context to chunks reduces retrieval failures by 49%. - **Code.** tree-sitter AST splitting via tree-sitter-language-pack for 150+ languages, with symbol name, type, and line range in chunk headers. -- **PDF.** kreuzberg text extraction with an OCR fallback chain (text extraction → Tesseract OCR → GGUF vision model on `llama-server`). PDF page rasterization is delegated to kreuzberg's `PdfPageIterator`. +- **PDF.** xberg text extraction with an OCR fallback chain (text extraction → Tesseract OCR → GGUF vision model on `llama-server`). PDF page rasterization is delegated to xberg's `PdfPageIterator`. - **Vision OCR.** When `LILBEE_VISION_MODEL` is set, scanned PDFs and images are transcribed by a GGUF vision model served by `llama-server` with an `--mmproj` projector. The pipeline streams an SSE heartbeat during long scans and preserves tables, headings, and multi-column layout as structured markdown. Falls back to Tesseract when no vision model is configured. -- **Structured files.** kreuzberg handles XML, JSON, JSONL, YAML, and CSV natively. Language detection for code-shaped content is delegated to tree-sitter-language-pack's `detect_language()`. +- **Structured files.** xberg handles XML, JSON, JSONL, YAML, and CSV natively. Language detection for code-shaped content is delegated to tree-sitter-language-pack's `detect_language()`. - **Web pages.** crawl4ai fetches HTML with JavaScript rendering via Playwright, converts to markdown, and saves to `documents/_web/` for indexing. Recursive crawls emit live progress, respect per-domain rate limits, and retry on HTTP 429/503 with jitter. SSRF protection blocks internal networks by default. -- **Chunking strategy.** Fixed-size chunking (default, token-aware) for reliability on procedural and reference docs. Opt-in semantic chunking (`LILBEE_SEMANTIC_CHUNKING=true`) splits at topic boundaries via kreuzberg's ONNX embedding model; better on prose-heavy corpora at the cost of roughly 9x more downstream embedding calls. +- **Chunking strategy.** Fixed-size chunking (default, token-aware) for reliability on procedural and reference docs. Opt-in semantic chunking (`LILBEE_SEMANTIC_CHUNKING=true`) splits at topic boundaries via xberg's ONNX embedding model; better on prose-heavy corpora at the cost of roughly 9x more downstream embedding calls. - **Embedding.** Provider-agnostic: native GGUF on the local `llama-server` engine by default, or any backend reachable via the SDK protocol when `pip install lilbee[litellm]` is available. - **Asymmetric query/document embedding.** Instruction-tuned embedders (Qwen3-Embedding, e5/gte `*-instruct`) only reach their retrieval scores when the query carries a task instruction (`Instruct: ...\nQuery: `) embedded differently from documents; base e5 uses `query:`/`passage:` prefixes. lilbee detects the family from the configured embedder ref and applies the right prefixes automatically (`retrieval/embedding_profiles.py`): the query path uses `embed_query`/`embed_query_batch`, the document path keeps `embed`/`embed_batch`. Symmetric models (bge-m3, nomic) and unrecognized models get no prefix, so behavior is unchanged for them. There is no instruction config: a wrong template silently underperforms, so support for a new family is a curated map entry, not a user knob. - **Concept extraction (opt-in).** With `pip install lilbee[graph]`, spaCy noun phrases are extracted per chunk, a co-occurrence graph is built with PPMI weights, and Leiden clustering assigns concepts to communities. diff --git a/docs/benchmarks/vision-ocr.md b/docs/benchmarks/vision-ocr.md index 2b88d7d33..774cf3cc6 100644 --- a/docs/benchmarks/vision-ocr.md +++ b/docs/benchmarks/vision-ocr.md @@ -34,7 +34,7 @@ Extract ALL text from this page as clean markdown. Preserve table structure. - **LightOnOCR-2 is the clear winner:** fastest, smallest, and produces the best-structured output with clean markdown formatting - **DeepSeek-OCR is the accuracy runner-up** with very clean text extraction but no markdown formatting - **Model size does not correlate with quality:** the 1B model beat the 8B model in both speed and accuracy -- **Vision OCR is only useful for scanned/image PDFs** — for text-based PDFs, Kreuzberg's text extraction is faster and more accurate +- **Vision OCR is only useful for scanned/image PDFs** — for text-based PDFs, Xberg's text extraction is faster and more accurate ## Limitations diff --git a/docs/usage.md b/docs/usage.md index 1496a2127..0df505d5f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1018,7 +1018,7 @@ chunk is more likely to contain the full section that matches rather than the first half of it plus unrelated setup. **Trade-off:** Enabling semantic chunking triggers a one-time download of -kreuzberg's ONNX embedding model (separate from the chunk-to-vector embedder) +xberg's ONNX embedding model (separate from the chunk-to-vector embedder) and runs roughly 9x more downstream embedding calls during indexing. Indexing takes longer; retrieval latency is unchanged. diff --git a/pyproject.toml b/pyproject.toml index 3c3d286df..6cf18fed0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ classifiers = [ dependencies = [ "lancedb", - "kreuzberg>=5.0.0rc35", + "xberg>=1.0.0rc1", "filelock", "tree-sitter-language-pack>=1.8.0,<2.0", "typer>=0.12", @@ -92,7 +92,9 @@ packages = ["src/lilbee"] # the index instead -- this source override is workspace metadata, not published deps. [tool.uv.sources] lilbee-engine = { path = "packaging/engine-wheel" } -kreuzberg = { path = "/Users/tobias/projects/kreuzberg/.worktrees/rc35/packages/python/dist/kreuzberg-5.0.0rc35-cp310-abi3-macosx_11_0_arm64.whl" } # LOCAL DEV ONLY (uncommitted) +# TODO: xberg 1.0.0rc1 is a pre-publication trial wheel; resolve it from the local +# build until it lands on PyPI, then drop this source and lock from the index. +xberg = { path = "/Users/tobias/projects/xberg-alef-trial-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" } [dependency-groups] dev = [ diff --git a/site/index.html b/site/index.html index 326406c43..67710be67 100644 --- a/site/index.html +++ b/site/index.html @@ -539,7 +539,7 @@

go deeper

built on

- Kreuzberg · + Xberg · llama.cpp · llama-swap · gguf-parser · diff --git a/src/lilbee/app/services.py b/src/lilbee/app/services.py index 4b7296737..95de70821 100644 --- a/src/lilbee/app/services.py +++ b/src/lilbee/app/services.py @@ -121,7 +121,7 @@ def get_services() -> Services: Service modules are imported inside the function to keep CLI startup fast: ``services`` is on every CLI import path, and the concrete service modules transitively pull in heavy libraries - (lancedb, kreuzberg). Deferring the loads until first + (lancedb, xberg). Deferring the loads until first ``get_services()`` call makes ``lilbee --help`` and TUI splash render in milliseconds instead of seconds. """ @@ -196,15 +196,15 @@ def get_services() -> Services: def sync_vision_ocr_backend(provider: LLMProvider) -> None: - """Register or unregister lilbee's vision model as kreuzberg's OCR backend. + """Register or unregister lilbee's vision model as xberg's OCR backend. Driven by ``cfg.vision_model``: registered while a model is set, removed when cleared. The backend reads ``cfg.vision_model`` live, so a model swap needs no re-registration, but it captures ``provider.vision_ocr``, so it must re-bind whenever the provider is rebuilt (``reset_services``) -- otherwise the global - kreuzberg registry keeps routing OCR to the shut-down provider. + xberg registry keeps routing OCR to the shut-down provider. """ - from kreuzberg import list_ocr_backends, register_ocr_backend, unregister_ocr_backend + from xberg import list_ocr_backends, register_ocr_backend, unregister_ocr_backend from lilbee.core.config import cfg from lilbee.data.ingest.types import OcrBackendName diff --git a/src/lilbee/app/settings.py b/src/lilbee/app/settings.py index 43d5d719d..e92fe4315 100644 --- a/src/lilbee/app/settings.py +++ b/src/lilbee/app/settings.py @@ -273,7 +273,7 @@ def _reload_changed_roles(changed_keys: set[str]) -> None: for field in changed_role_fields: services.reload_role(MODEL_FIELD_TO_ROLE[field]) if "vision_model" in changed_role_fields: - # Register/unregister lilbee's kreuzberg OCR backend on any vision-model + # Register/unregister lilbee's xberg OCR backend on any vision-model # change (REST/MCP/TUI/CLI all funnel here), not just the REST route. from lilbee.app.services import sync_vision_ocr_backend diff --git a/src/lilbee/cli/tui/__init__.py b/src/lilbee/cli/tui/__init__.py index 5a702d125..60d3d13fa 100644 --- a/src/lilbee/cli/tui/__init__.py +++ b/src/lilbee/cli/tui/__init__.py @@ -45,14 +45,14 @@ def _redirect_native_stderr_to(log_path: Path) -> _StderrRedirect | None: Textual's Linux/macOS driver writes its alternate-screen ANSI to ``sys.__stderr__`` (see ``textual.drivers.linux_driver.LinuxDriver.write``). Native deps - like kreuzberg's vendored tesseract write directly to fd 2 and leak + like xberg's vendored tesseract write directly to fd 2 and leak onto the same buffer, e.g. "Detected N diacritics", which corrupts the TUI. Strategy: dup the original fd 2 to a saved fd, repoint ``sys.__stderr__`` and ``sys.stderr`` at that saved fd so Textual keeps drawing to the real terminal, then dup2 fd 2 itself to the - log file so any fd-2 writer (kreuzberg, tesseract, poppler) lands + log file so any fd-2 writer (xberg, tesseract, poppler) lands in ``tui.log`` instead of on top of the screen. """ try: diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index 6d15cd0e4..45bd6d56b 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -134,7 +134,7 @@ class Config(BaseSettings): tesseract_timeout: float = ConfigField(default=60.0, ge=0.0, writable=True) # Tesseract OCR language codes for the scanned-document fallback (used when no # vision model is set), e.g. ["eng"] or ["eng", "deu"]. Set via env as - # LILBEE_OCR_LANGUAGE="eng+deu". kreuzberg requires a non-empty list. + # LILBEE_OCR_LANGUAGE="eng+deu". xberg requires a non-empty list. ocr_language: list[str] = ConfigField(default_factory=lambda: ["eng"], writable=True) semantic_chunking: bool = ConfigField(default=False, writable=True) topic_threshold: float = ConfigField(default=0.75, ge=0.0, le=1.0, writable=True) @@ -687,7 +687,7 @@ def _parse_enable_ocr(cls, v: Any) -> bool | None: def _parse_ocr_language(cls, v: Any) -> list[str]: """Accept a list or a comma/plus-separated string; never return empty. - Tesseract uses ``+`` to join languages and kreuzberg errors on an empty + Tesseract uses ``+`` to join languages and xberg errors on an empty list, so blank input falls back to English. """ items = v.replace("+", ",").split(",") if isinstance(v, str) else (v or []) diff --git a/src/lilbee/data/chunk.py b/src/lilbee/data/chunk.py index 1b9f13071..ff11e1b6c 100644 --- a/src/lilbee/data/chunk.py +++ b/src/lilbee/data/chunk.py @@ -8,13 +8,13 @@ from lilbee.core.config import cfg if TYPE_CHECKING: - from kreuzberg import ChunkingConfig + from xberg import ChunkingConfig CHARS_PER_TOKEN = 4 _SEMANTIC_CHUNKER = "semantic" _MARKDOWN_CHUNKER = "markdown" -# Kreuzberg silently falls back to a non-semantic path when embedding is None. +# Xberg silently falls back to a non-semantic path when embedding is None. _SEMANTIC_EMBEDDING_PRESET = "fast" _DISABLE_PROGRESS_ENV = "HF_HUB_DISABLE_PROGRESS_BARS" @@ -37,8 +37,8 @@ def _show_download_progress() -> bool: def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: - """Build a kreuzberg ChunkingConfig from the current cfg.""" - from kreuzberg import ChunkingConfig, EmbeddingConfig + """Build an xberg ChunkingConfig from the current cfg.""" + from xberg import ChunkingConfig, EmbeddingConfig, EmbeddingModelType max_chars, max_overlap = _char_budget() @@ -46,7 +46,9 @@ def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: return ChunkingConfig( chunker_type=_SEMANTIC_CHUNKER, embedding=EmbeddingConfig( - model=_SEMANTIC_EMBEDDING_PRESET, + # xberg's .pyi omits the per-variant constructors alef #147 added at + # runtime; preset() works but isn't declared in the stub yet. + model=EmbeddingModelType.preset(_SEMANTIC_EMBEDDING_PRESET), # type: ignore[attr-defined] show_download_progress=_show_download_progress(), ), topic_threshold=cfg.topic_threshold, @@ -67,7 +69,7 @@ def chunk_text( if not text or not text.strip(): return [] - from kreuzberg import ChunkingConfig, ExtractionConfig, extract_bytes_sync + from xberg import ChunkingConfig, ExtractionConfig, extract_bytes_sync if heading_context: max_chars, max_overlap = _char_budget() diff --git a/src/lilbee/data/ingest/discovery.py b/src/lilbee/data/ingest/discovery.py index c638f302c..a181fcdd4 100644 --- a/src/lilbee/data/ingest/discovery.py +++ b/src/lilbee/data/ingest/discovery.py @@ -20,7 +20,7 @@ def _content_type_for(ext: str, mime: str) -> str: - """content_type for a kreuzberg format: PDFs and images grouped, others keyed by extension.""" + """content_type for a xberg format: PDFs and images grouped, others keyed by extension.""" if mime == _PDF_MIME: return PDF_CONTENT_TYPE if mime.startswith("image/"): @@ -30,13 +30,13 @@ def _content_type_for(ext: str, mime: str) -> str: @cache def supported_extension_map() -> dict[str, str]: - """Extension -> content_type for every format kreuzberg can extract. + """Extension -> content_type for every format xberg can extract. - Built from ``kreuzberg.list_supported_formats()`` so lilbee covers the full set + Built from ``xberg.list_supported_formats()`` so lilbee covers the full set without a hand-maintained list. Source-code files are routed separately (their extensions are absent here), so ``classify_file`` falls through to the code path. """ - from kreuzberg import list_supported_formats + from xberg import list_supported_formats out: dict[str, str] = {} for fmt in list_supported_formats(): @@ -60,9 +60,9 @@ def _relative_name(path: Path) -> str: def classify_file(path: Path) -> str | None: - """Classify a file by extension: a kreuzberg content_type, "code", or None. + """Classify a file by extension: a xberg content_type, "code", or None. - kreuzberg-extractable formats win; source code (not in kreuzberg's set) routes + xberg-extractable formats win; source code (not in xberg's set) routes to the code chunker; anything else is unsupported. """ doc_type = supported_extension_map().get(path.suffix.lower()) diff --git a/src/lilbee/data/ingest/extract.py b/src/lilbee/data/ingest/extract.py index 0839e2f26..1a952bfa7 100644 --- a/src/lilbee/data/ingest/extract.py +++ b/src/lilbee/data/ingest/extract.py @@ -1,4 +1,4 @@ -"""Document extraction: one kreuzberg pass that natively extracts text and OCRs +"""Document extraction: one xberg pass that natively extracts text and OCRs scanned pages/images through the registered backend; chunk + embed the result.""" from __future__ import annotations @@ -32,11 +32,11 @@ ) if TYPE_CHECKING: - from kreuzberg import ExtractionConfig, OcrConfig + from xberg import ExtractionConfig, OcrConfig # extract_* return the pyo3 result (attribute access), not the public - # ExtractionResult TypedDict (kreuzberg-7ih). - from kreuzberg._kreuzberg import ExtractionResult + # ExtractionResult TypedDict (xberg-7ih). + from xberg._xberg import ExtractionResult log = logging.getLogger(__name__) @@ -103,26 +103,26 @@ def _ocr_config(ocr_token: str | None) -> OcrConfig: """Pick the OCR backend for this extraction. Mirrors the prior fallback policy: OCR off when ``enable_ocr`` is False; lilbee's - vision backend when a vision model is configured; otherwise kreuzberg's tesseract. - kreuzberg auto-OCRs only the pages that lack a text layer. + vision backend when a vision model is configured; otherwise xberg's tesseract. + xberg auto-OCRs only the pages that lack a text layer. """ - from kreuzberg import OcrConfig + from xberg import OcrConfig if _effective_enable_ocr() is False: return OcrConfig(enabled=False) if cfg.vision_model: options = backend_options_for(ocr_token) if ocr_token else None return OcrConfig(backend=OcrBackendName.LILBEE_VISION, backend_options=options) - # kreuzberg requires a non-empty language list (4.x defaulted to English; + # xberg requires a non-empty language list (4.x defaulted to English; # 5.x errors on an empty one). cfg.ocr_language is validated non-empty. return OcrConfig(backend=OcrBackendName.TESSERACT, language=list(cfg.ocr_language)) def extraction_config(mode: ExtractMode, *, ocr_token: str | None = None) -> ExtractionConfig: """Build ExtractionConfig for the given extraction mode.""" - from kreuzberg import ExtractionConfig, PageConfig + from xberg import ExtractionConfig, PageConfig - # Files are extracted one per call, so kreuzberg parallelizes OCR across the + # Files are extracted one per call, so xberg parallelizes OCR across the # pages of each document internally; cross-file concurrency is the pipeline's # semaphore. (max_concurrent_extractions only bounds multi-file batch calls, # which lilbee never makes, so it is intentionally not set here.) @@ -161,7 +161,7 @@ async def chunk_and_embed_pages( if not page_texts: return [] - # chunk_text runs kreuzberg's synchronous extractor; offload it so a long + # chunk_text runs xberg's synchronous extractor; offload it so a long # document does not stall sibling files sharing this event loop. all_chunks = await asyncio.to_thread(_chunk_pages, page_texts) if not all_chunks: @@ -229,15 +229,15 @@ async def ingest_document( on_progress: DetailedProgressCallback = noop_callback, page_texts_out: list[PageTextRecord] | None = None, ) -> list[ChunkRecord]: - """Extract, chunk, and embed a document in a single kreuzberg pass. + """Extract, chunk, and embed a document in a single xberg pass. - kreuzberg extracts native text and, where a page has none, OCRs it through the + xberg extracts native text and, where a page has none, OCRs it through the registered backend (lilbee's vision model, or tesseract). Per-page OCR progress is streamed as a running count via ``ocr_request``. ``quiet`` is accepted for pipeline call compatibility. """ del quiet - from kreuzberg import extract_file + from xberg import extract_file page_seen = 0 @@ -306,7 +306,7 @@ async def ingest_markdown( if not raw_text.strip(): return [] - # chunk_text runs kreuzberg's synchronous extractor; offload it so a large + # chunk_text runs xberg's synchronous extractor; offload it so a large # markdown doc does not stall sibling files sharing this event loop. texts = await asyncio.to_thread( chunk_text, raw_text, mime_type="text/markdown", heading_context=True diff --git a/src/lilbee/data/ingest/types.py b/src/lilbee/data/ingest/types.py index 96c923906..f2bc7e5ea 100644 --- a/src/lilbee/data/ingest/types.py +++ b/src/lilbee/data/ingest/types.py @@ -48,7 +48,7 @@ class FileChangePlan(NamedTuple): class OcrBackendName(StrEnum): - """OCR backends lilbee selects in OcrConfig: kreuzberg's tesseract or lilbee's vision plugin.""" + """OCR backends lilbee selects in OcrConfig: xberg's tesseract or lilbee's vision plugin.""" TESSERACT = "tesseract" LILBEE_VISION = "lilbee-vision" diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py index 781c81547..3967de7d1 100644 --- a/src/lilbee/data/ingest/vision_ocr_backend.py +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -1,4 +1,4 @@ -"""lilbee's vision model exposed as a kreuzberg custom OCR backend.""" +"""lilbee's vision model exposed as a xberg custom OCR backend.""" from __future__ import annotations @@ -16,22 +16,22 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator -# Token key inside OcrConfig.backend_options JSON. kreuzberg does not propagate -# contextvars into process_image (kreuzberg-4w9), so per-request state travels as +# Token key inside OcrConfig.backend_options JSON. xberg does not propagate +# contextvars into process_image (xberg-4w9), so per-request state travels as # a token on the config and is resolved through the registry below. _REQUEST_TOKEN_KEY = "req" # noqa: S105 # JSON key name, not a secret @dataclass(frozen=True) class OcrRequestContext: - """Per-extraction state the backend needs but kreuzberg won't carry for it.""" + """Per-extraction state the backend needs but xberg won't carry for it.""" on_page: Callable[[], None] | None = None timeout: float = 0.0 class _OcrRequestRegistry: - """Token-keyed request contexts; lock-guarded (process_image runs on kreuzberg threads).""" + """Token-keyed request contexts; lock-guarded (process_image runs on xberg threads).""" def __init__(self) -> None: self._lock = threading.Lock() @@ -75,7 +75,7 @@ def backend_options_for(token: str) -> str: def _config_to_dict(config: str) -> dict[str, Any]: - """Parse the OcrConfig JSON string kreuzberg passes to process_image into a dict.""" + """Parse the OcrConfig JSON string xberg passes to process_image into a dict.""" try: raw: object = json.loads(config) except (ValueError, TypeError): @@ -84,7 +84,7 @@ def _config_to_dict(config: str) -> dict[str, Any]: class _OcrConfigView: - """Typed reader over the kreuzberg OcrConfig (normalized to a dict).""" + """Typed reader over the xberg OcrConfig (normalized to a dict).""" def __init__(self, config: dict[str, Any]) -> None: self._config = config @@ -122,7 +122,7 @@ def _lilbee_version() -> str: class VisionOcrBackend: - """Routes kreuzberg OCR calls to lilbee's vision model through the injected + """Routes xberg OCR calls to lilbee's vision model through the injected ``ocr_fn`` (single-image OCR) and ``model_ref_fn`` (the active vision model).""" def __init__(self, *, ocr_fn: _OcrFn, model_ref_fn: Callable[[], str]) -> None: @@ -149,7 +149,7 @@ def backend_type(self) -> str: return "custom" def process_image(self, image_bytes: bytes, config: str) -> dict[str, Any]: - # kreuzberg serializes the OcrConfig to a JSON string for the callback; + # xberg serializes the OcrConfig to a JSON string for the callback; # parse before reading fields. view = _OcrConfigView(_config_to_dict(config)) model = self._model_ref_fn() @@ -158,8 +158,8 @@ def process_image(self, image_bytes: bytes, config: str) -> dict[str, Any]: text = self._ocr_fn(image_bytes, model, prompt, timeout=ctx.timeout if ctx else 0.0) if ctx is not None and ctx.on_page is not None: ctx.on_page() - # kreuzberg deserializes this dict into its OCR result struct; all of these - # keys are required (kreuzberg-1mc). + # xberg deserializes this dict into its OCR result struct; all of these + # keys are required (xberg-1mc). return { "content": text, "mime_type": MARKDOWN_MIME, diff --git a/src/lilbee/providers/fleet/provider.py b/src/lilbee/providers/fleet/provider.py index e8e1e8b59..bafbd3415 100644 --- a/src/lilbee/providers/fleet/provider.py +++ b/src/lilbee/providers/fleet/provider.py @@ -180,7 +180,7 @@ def _supports_tools_cached(path_str: str, _mtime_ns: int) -> bool: class _VisionRequestGate: """Process-wide cap on concurrent vision-server requests at the fleet's OCR slots. - The ingest file fan-out runs many files at once and kreuzberg OCRs their pages + The ingest file fan-out runs many files at once and xberg OCRs their pages through per-image ``vision_ocr`` calls, so without a shared cap the aggregate over-subscribes a single-replica vision server into a 429 storm. The semaphore is rebuilt to the configured capacity (``vision_replicas * vision_ocr_concurrency``) @@ -641,7 +641,7 @@ def vision_ocr( clients, lambda client: _vision_call(client, messages, timeout) ) - # PDF/image OCR now runs inside kreuzberg via the registered lilbee-vision + # PDF/image OCR now runs inside xberg via the registered lilbee-vision # backend (see data.ingest.vision_ocr_backend); this provider only exposes # single-image vision_ocr, which that backend calls. diff --git a/src/lilbee/runtime/progress/types.py b/src/lilbee/runtime/progress/types.py index 86256e96f..a89952385 100644 --- a/src/lilbee/runtime/progress/types.py +++ b/src/lilbee/runtime/progress/types.py @@ -83,7 +83,7 @@ class BatchProgressEvent(BaseModel): class ExtractEvent(BaseModel): """Emitted with page-level extraction progress. - OCR fires one event per page as kreuzberg processes it, as a running count + OCR fires one event per page as xberg processes it, as a running count with ``total_pages == 0`` (the total is unknown mid-extraction). Extraction then fires once per file with ``page == total_pages`` so subscribers see "extracted N pages" before the embed phase ticks. diff --git a/src/lilbee/skills/lilbee_mcp/SKILL.md b/src/lilbee/skills/lilbee_mcp/SKILL.md index e7b6ea7d4..87c670710 100644 --- a/src/lilbee/skills/lilbee_mcp/SKILL.md +++ b/src/lilbee/skills/lilbee_mcp/SKILL.md @@ -175,7 +175,7 @@ them back. ### 5. Your first answer feels thin -- self-tune and retry When the user asks a broad question against a dense pile of reference -docs (godot class XMLs, an API reference, kreuzberg-style docstrings) +docs (godot class XMLs, an API reference, xberg-style docstrings) and your first `lilbee_search` returns only one or two relevant hits where you'd expect a family, the retrieval defaults are too narrow for the shape of what's indexed. Self-tune in-place rather than handing the diff --git a/src/lilbee/vision.py b/src/lilbee/vision.py index 67079f2a3..b0f2e4c27 100644 --- a/src/lilbee/vision.py +++ b/src/lilbee/vision.py @@ -1,6 +1,6 @@ """Vision-model OCR helpers: prompt resolution and OpenAI-compatible image messages. -PDF rasterisation and the page loop now live inside kreuzberg (the registered +PDF rasterisation and the page loop now live inside xberg (the registered lilbee-vision OCR backend); this module only builds the single-image request the provider's ``vision_ocr`` sends to the vision server. """ diff --git a/tests/integration/test_pdf_integration.py b/tests/integration/test_pdf_integration.py index 9d5905cfe..dbff8fc8d 100644 --- a/tests/integration/test_pdf_integration.py +++ b/tests/integration/test_pdf_integration.py @@ -3,7 +3,7 @@ Uses a rasterized PDF fixture (no selectable text) to exercise the full extraction pipeline including Tesseract OCR and vision model fallbacks. -Requires: kreuzberg, Tesseract (for OCR tests), a vision model (for vision tests). +Requires: xberg, Tesseract (for OCR tests), a vision model (for vision tests). Skipped automatically when dependencies are not available. Run with: @@ -132,7 +132,7 @@ class TestTesseractOcrFallback: ) async def test_tesseract_extracts_text(self): """Tesseract OCR produces non-empty text from the scanned PDF fixture.""" - from kreuzberg import ExtractionConfig, OcrConfig, extract_file + from xberg import ExtractionConfig, OcrConfig, extract_file config = ExtractionConfig(ocr=OcrConfig(backend="tesseract"), force_ocr=True) result = await extract_file(str(SCANNED_PDF), config=config) @@ -144,7 +144,7 @@ async def test_tesseract_extracts_text(self): ) async def test_tesseract_extracts_known_phrases(self): """Tesseract OCR captures key phrases from the scanned document.""" - from kreuzberg import ExtractionConfig, OcrConfig, extract_file + from xberg import ExtractionConfig, OcrConfig, extract_file config = ExtractionConfig(ocr=OcrConfig(backend="tesseract"), force_ocr=True) result = await extract_file(str(SCANNED_PDF), config=config) @@ -171,10 +171,10 @@ def _vision_model_available() -> bool: class TestVisionOcrFallback: - """Vision OCR through the registered lilbee-vision kreuzberg backend.""" + """Vision OCR through the registered lilbee-vision xberg backend.""" async def _vision_extract(self) -> str: - from kreuzberg import ExtractionConfig, OcrConfig, extract_file + from xberg import ExtractionConfig, OcrConfig, extract_file from lilbee.app.services import get_services, sync_vision_ocr_backend from lilbee.data.ingest.types import OcrBackendName diff --git a/tests/server/test_handlers.py b/tests/server/test_handlers.py index d410863b6..3257c5773 100644 --- a/tests/server/test_handlers.py +++ b/tests/server/test_handlers.py @@ -76,7 +76,7 @@ def reset_ingest_locks(): get_services().ingest_lock_registry.reset() -def _make_kreuzberg_result(text: str = "Some extracted text. " * 20, num_chunks: int = 1): +def _make_xberg_result(text: str = "Some extracted text. " * 20, num_chunks: int = 1): chunks = [] for i in range(num_chunks): chunk_text = text[i * len(text) // num_chunks : (i + 1) * len(text) // num_chunks] @@ -91,7 +91,7 @@ def _make_kreuzberg_result(text: str = "Some extracted text. " * 20, num_chunks: return result -@mock.patch("kreuzberg.extract_file_sync", new_callable=Mock, return_value=_make_kreuzberg_result()) +@mock.patch("xberg.extract_file_sync", new_callable=Mock, return_value=_make_xberg_result()) class TestAddEndpoint: async def test_add_single_file(self, mock_extract_file, isolated_env, tmp_path): """POST /api/add with a valid file streams SSE events and adds it.""" @@ -271,9 +271,9 @@ async def test_exactly_max_files_accepted(self, isolated_env, tmp_path): paths = [f"/fake/file_{i}.txt" for i in range(MAX_ADD_FILES)] with mock.patch( - "kreuzberg.extract_file_sync", + "xberg.extract_file_sync", new_callable=Mock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ): async with AsyncTestClient(create_app()) as client: resp = await client.post("/api/add", json={"paths": paths}, headers=_auth_headers()) @@ -537,9 +537,9 @@ async def test_partial_contention_partitions_paths(self, isolated_env, tmp_path) assert lock is not None try: with mock.patch( - "kreuzberg.extract_file_sync", + "xberg.extract_file_sync", new_callable=Mock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ): events = await self._collect(add_files_stream([str(held), str(free)])) finally: @@ -567,9 +567,9 @@ async def test_concurrent_different_sources_run_in_parallel(self, isolated_env, async def _run(path: Path): text = "" with mock.patch( - "kreuzberg.extract_file_sync", + "xberg.extract_file_sync", new_callable=Mock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ): async for frame in add_files_stream([str(path)]): text += frame @@ -652,9 +652,9 @@ async def test_new_file_triggers_cleanup(self, isolated_env, tmp_path, mock_svc) store.get_sources.return_value = [] with mock.patch( - "kreuzberg.extract_file_sync", + "xberg.extract_file_sync", new_callable=Mock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ): await sync(quiet=True) @@ -676,9 +676,9 @@ async def test_retry_after_orphaned_chunks_cleans_up(self, isolated_env, tmp_pat store.get_sources.return_value = [] with mock.patch( - "kreuzberg.extract_file_sync", + "xberg.extract_file_sync", new_callable=Mock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ): await sync(quiet=True) diff --git a/tests/test_chunker.py b/tests/test_chunker.py index e38bcabf6..8dcb534c3 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -106,7 +106,7 @@ def test_use_semantic_false_bypasses_semantic(self, monkeypatch): class TestBuildChunkingConfig: def test_semantic_enabled_uses_semantic_chunker_with_embedding(self, monkeypatch): - """Semantic path requires an EmbeddingConfig or kreuzberg silently falls back.""" + """Semantic path requires an EmbeddingConfig or xberg silently falls back.""" from lilbee.core.config import cfg from lilbee.data.chunk import build_chunking_config @@ -118,7 +118,7 @@ def test_semantic_enabled_uses_semantic_chunker_with_embedding(self, monkeypatch assert result.embedding is not None def test_semantic_respects_max_chars_when_embedding_present(self, monkeypatch): - """With an embedding attached kreuzberg honors max_characters on the semantic path.""" + """With an embedding attached xberg honors max_characters on the semantic path.""" from lilbee.core.config import cfg from lilbee.data.chunk import CHARS_PER_TOKEN, build_chunking_config @@ -127,7 +127,10 @@ def test_semantic_respects_max_chars_when_embedding_present(self, monkeypatch): result = build_chunking_config() assert result.max_characters == 512 * CHARS_PER_TOKEN assert result.embedding is not None - assert result.embedding.model == "fast" + # Built via the real EmbeddingModelType.preset() constructor (xberg/alef #147), + # not the bare-string shorthand. + assert result.embedding.model.type == "preset" + assert str(result.embedding.model) == '{"type":"preset","name":"fast"}' def test_char_budget_when_disabled(self, monkeypatch): from lilbee.core.config import cfg @@ -592,7 +595,7 @@ def test_real_parser_line_range_spans_no_trailing_newline_file(self): class TestHeadingContextNoDuplicate: def test_heading_context_no_duplicate(self): - """kreuzberg >= 4.8.5 should not duplicate headings with prepend_heading_context.""" + """xberg >= 4.8.5 should not duplicate headings with prepend_heading_context.""" md = "# Title\n\n" + "Word " * 500 + "\n\n## Section\n\n" + "More " * 500 chunks = chunk_text(md, mime_type="text/markdown", heading_context=True) for c in chunks: @@ -610,5 +613,5 @@ def test_returns_empty_when_no_chunks(self): mock_result = MagicMock() mock_result.chunks = [] - with patch("kreuzberg.extract_bytes_sync", return_value=mock_result): + with patch("xberg.extract_bytes_sync", return_value=mock_result): assert chunk_text("some text") == [] diff --git a/tests/test_dynamic_settings_eviction.py b/tests/test_dynamic_settings_eviction.py index e4d8828f9..4c6233e23 100644 --- a/tests/test_dynamic_settings_eviction.py +++ b/tests/test_dynamic_settings_eviction.py @@ -284,10 +284,10 @@ def test_chat_and_vision_model_change_reloads_those_roles(monkeypatch): from lilbee.providers.roles import WorkerRole # The vision-model reload also (un)registers the OCR backend; keep that global - # kreuzberg state out of this unit test. - monkeypatch.setattr("kreuzberg.list_ocr_backends", list) - monkeypatch.setattr("kreuzberg.register_ocr_backend", lambda backend: None) - monkeypatch.setattr("kreuzberg.unregister_ocr_backend", lambda name: None) + # xberg state out of this unit test. + monkeypatch.setattr("xberg.list_ocr_backends", list) + monkeypatch.setattr("xberg.register_ocr_backend", lambda backend: None) + monkeypatch.setattr("xberg.unregister_ocr_backend", lambda name: None) provider = _install_recording_provider() try: diff --git a/tests/test_formats.py b/tests/test_formats.py index c9eee1446..f19ceec70 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -1,8 +1,8 @@ """Real-file format tests: full sync() pipeline with actual files on disk. -All document formats go through kreuzberg. Code files still use tree-sitter. -Embeddings are mocked (no live LLM server needed). kreuzberg is mocked for document -extraction since we're testing the pipeline, not kreuzberg itself. +All document formats go through xberg. Code files still use tree-sitter. +Embeddings are mocked (no live LLM server needed). xberg is mocked for document +extraction since we're testing the pipeline, not xberg itself. """ from __future__ import annotations @@ -49,8 +49,8 @@ def mock_svc(): set_services(None) -def _make_kreuzberg_result(text="Extracted content. " * 10, num_chunks=1): - """Build a mock kreuzberg ExtractionResult.""" +def _make_xberg_result(text="Extracted content. " * 10, num_chunks=1): + """Build a mock xberg ExtractionResult.""" chunks = [] for i in range(num_chunks): chunk_text = text[i * len(text) // num_chunks : (i + 1) * len(text) // num_chunks] @@ -74,14 +74,14 @@ def _make_kreuzberg_result(text="Extracted content. " * 10, num_chunks=1): # --------------------------------------------------------------------------- -# Document formats (all go through kreuzberg) +# Document formats (all go through xberg) # --------------------------------------------------------------------------- @mock.patch( - "kreuzberg.extract_file", + "xberg.extract_file", new_callable=mock.AsyncMock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ) class TestSyncDocx: async def test_docx_discovered_and_ingested(self, mock_extract_file, isolated_env): @@ -93,9 +93,9 @@ async def test_docx_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "kreuzberg.extract_file", + "xberg.extract_file", new_callable=mock.AsyncMock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ) class TestSyncXlsx: async def test_xlsx_discovered_and_ingested(self, mock_extract_file, isolated_env): @@ -107,9 +107,9 @@ async def test_xlsx_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "kreuzberg.extract_file", + "xberg.extract_file", new_callable=mock.AsyncMock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ) class TestSyncPptx: async def test_pptx_discovered_and_ingested(self, mock_extract_file, isolated_env): @@ -126,9 +126,9 @@ async def test_pptx_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "kreuzberg.extract_file", + "xberg.extract_file", new_callable=mock.AsyncMock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ) class TestSyncEpub: async def test_epub_discovered_and_ingested(self, mock_extract_file, isolated_env): @@ -145,9 +145,9 @@ async def test_epub_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "kreuzberg.extract_file", + "xberg.extract_file", new_callable=mock.AsyncMock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ) class TestSyncImage: async def test_image_discovered_and_ingested(self, mock_extract_file, isolated_env): @@ -224,9 +224,9 @@ async def test_code_file_syncs(self, isolated_env, filename, fixture): @mock.patch( - "kreuzberg.extract_file", + "xberg.extract_file", new_callable=mock.AsyncMock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ) class TestSyncCsvTsv: async def test_csv_discovered_and_ingested(self, mock_extract_file, isolated_env): diff --git a/tests/test_ingest.py b/tests/test_ingest.py index af0aed4ab..21b5ebece 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -132,13 +132,13 @@ def _real_ingest_result(name, *, file_hash, page_text="page one of a.pdf", stat= ) -def _make_kreuzberg_result( +def _make_xberg_result( text="Some extracted text. " * 20, num_chunks=1, has_pages=False, document=None, ): - """Build a mock kreuzberg ExtractionResult.""" + """Build a mock xberg ExtractionResult.""" chunks = [] for i in range(num_chunks): chunk_text = text[i * len(text) // num_chunks : (i + 1) * len(text) // num_chunks] @@ -168,7 +168,7 @@ def _make_kreuzberg_result( def _make_empty_result(): - """Build a mock kreuzberg ExtractionResult with no chunks.""" + """Build a mock xberg ExtractionResult with no chunks.""" result = mock.MagicMock() result.chunks = [] result.content = "" @@ -177,7 +177,7 @@ def _make_empty_result(): @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) class TestSync: async def test_empty_documents_dir(self, mock_extract_file, isolated_env): @@ -378,7 +378,7 @@ async def test_midstream_flush_when_chunk_threshold_crossed( def test_flush_writes_marks_files_failed_on_write_error(self, _mock_extract_file): # A failed batch write moves every buffered file to ``failed`` and out of # added/updated, since none of its chunks persisted; the buffer still clears. - # ``_mock_extract_file`` is the class-level kreuzberg patch, unused here. + # ``_mock_extract_file`` is the class-level xberg patch, unused here. from lilbee.app.services import get_services from lilbee.data.ingest import pipeline from lilbee.data.ingest.types import _IngestResult @@ -684,7 +684,7 @@ async def _fail(path, name, ct, **kwargs): @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) class TestSyncCancellation: """Tests for cancel support and atomic per-file delete in sync.""" @@ -828,7 +828,7 @@ class TestIngestHelpers: """Cover edge cases in ingest_document and ingest_code_sync.""" @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_empty_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_empty_result() ) async def testingest_document_empty_chunks(self, mock_extract_file, isolated_env): """Document that produces no chunks returns empty list.""" @@ -883,10 +883,10 @@ class _FakeResult: assert "# File: pkg/mod.py" in joined assert str(f) not in joined # absolute path must never leak into content - @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) async def testingest_document_pdf_with_pages(self, mock_kf, isolated_env): """PDF document returns records with page metadata.""" - mock_kf.return_value = _make_kreuzberg_result( + mock_kf.return_value = _make_xberg_result( text="Page 1 content. " * 10 + "Page 2 content. " * 10, num_chunks=2, has_pages=True, @@ -903,7 +903,7 @@ async def testingest_document_pdf_with_pages(self, mock_kf, isolated_env): class TestCancellation: @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_cancelled_error_propagates(self, mock_extract_file, isolated_env): """CancelledError in _process_one is re-raised, not swallowed.""" @@ -924,7 +924,7 @@ async def _cancel(*args, **kwargs): await ingest_batch([entry], added, {}, {}, {}, quiet=True) @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_task_cancelled_error_does_not_orphan_siblings( self, mock_extract_file, isolated_env @@ -1320,9 +1320,9 @@ def _spy_plan(*args, **kwargs): monkeypatch.setattr(pipeline, "_plan_file_changes", _spy_plan) with mock.patch( - "kreuzberg.extract_file", + "xberg.extract_file", new_callable=mock.AsyncMock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ): await sync(quiet=True) assert plan_threads @@ -1337,9 +1337,9 @@ async def test_sync_backfills_stats_via_store(self, isolated_env, mock_svc): mock_svc.store.upsert_source("legacy.txt", file_hash(f), 1, source_type="document") with mock.patch( - "kreuzberg.extract_file", + "xberg.extract_file", new_callable=mock.AsyncMock, - return_value=_make_kreuzberg_result(), + return_value=_make_xberg_result(), ): result = await sync(quiet=True) assert result.unchanged == 1 @@ -1946,9 +1946,9 @@ def test_paginated_has_tesseract_ocr_backend(self): config = extraction_config(ExtractMode.PAGINATED) assert config.get("pages") is not None assert config.get("ocr") is not None - # No vision model configured -> kreuzberg's tesseract backend OCRs scanned pages. + # No vision model configured -> xberg's tesseract backend OCRs scanned pages. assert config["ocr"].backend == "tesseract" - # kreuzberg 5.x errors on image OCR with an empty language list; lilbee must + # xberg 5.x errors on image OCR with an empty language list; lilbee must # set an explicit default to preserve 4.x behavior (regression guard). assert config["ocr"].language == ["eng"] @@ -1991,8 +1991,8 @@ def test_topic_threshold_propagates_to_every_mode(self, monkeypatch): assert config["chunking"].topic_threshold == pytest.approx(0.42, abs=1e-5) -class TestClassifyKreuzbergParityFormats: - """Formats kreuzberg extracts that the map must not silently drop.""" +class TestClassifyXbergParityFormats: + """Formats xberg extracts that the map must not silently drop.""" @pytest.mark.parametrize( "filename, expected", @@ -2012,7 +2012,7 @@ class TestClassifyKreuzbergParityFormats: ("mail.eml", "eml"), ("mail.msg", "msg"), ("table.dbf", "dbf"), - # Containers are supported too (kreuzberg extracts their combined text). + # Containers are supported too (xberg extracts their combined text). ("archive.pst", "pst"), ("archive.7z", "7z"), ], @@ -2042,7 +2042,7 @@ def test_classify(self, filename, expected): @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) class TestSyncStructuredFormats: async def test_xml_file_ingested( @@ -2252,9 +2252,9 @@ async def test_frontmatter_only_produces_chunks(self, isolated_env): class TestPageTextAccumulator: """`page_texts_out` captures clean per-page text for the export dataset.""" - @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) async def test_pdf_pages_captured(self, mock_kf, isolated_env): - mock_kf.return_value = _make_kreuzberg_result(num_chunks=2, has_pages=True) + mock_kf.return_value = _make_xberg_result(num_chunks=2, has_pages=True) from lilbee.data.ingest import ingest_document f = isolated_env / "test.pdf" @@ -2264,9 +2264,9 @@ async def test_pdf_pages_captured(self, mock_kf, isolated_env): assert [p["page"] for p in pages] == [1, 2] assert all(p["content_type"] == "pdf" for p in pages) - @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) async def test_non_paginated_doc_captured_as_page_zero(self, mock_kf, isolated_env): - mock_kf.return_value = _make_kreuzberg_result(text="Plain body. " * 10, has_pages=False) + mock_kf.return_value = _make_xberg_result(text="Plain body. " * 10, has_pages=False) from lilbee.data.ingest import ingest_document f = isolated_env / "note.txt" @@ -2289,12 +2289,12 @@ async def test_markdown_captured_as_page_zero(self, isolated_env): assert pages[0]["page"] == 0 assert "markdown body" in pages[0]["text"] - @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) async def test_ocr_pages_captured(self, mock_kf, isolated_env, mock_svc): - # kreuzberg OCRs scanned pages in-pass; the OCR'd text arrives in result.pages. + # xberg OCRs scanned pages in-pass; the OCR'd text arrives in result.pages. cfg.enable_ocr = True cfg.vision_model = "" - mock_kf.return_value = _make_kreuzberg_result( + mock_kf.return_value = _make_xberg_result( text="OCR page text. " * 10, num_chunks=2, has_pages=True ) from lilbee.data.ingest import ingest_document @@ -2309,12 +2309,12 @@ async def test_ocr_pages_captured(self, mock_kf, isolated_env, mock_svc): class TestIngestDocumentEdgeCases: async def test_empty_extraction_returns_empty(self, isolated_env): - """Structured formats now go through kreuzberg: empty result yields no chunks.""" + """Structured formats now go through xberg: empty result yields no chunks.""" from lilbee.data.ingest import ingest_document empty_result = mock.MagicMock(chunks=[]) mock_extract = mock.AsyncMock(return_value=empty_result) - with mock.patch("kreuzberg.extract_file", mock_extract): + with mock.patch("xberg.extract_file", mock_extract): result = await ingest_document(isolated_env / "e.xml", "e.xml", "xml") assert result == [] @@ -2323,12 +2323,12 @@ async def test_no_chunks_returns_empty(self, isolated_env): no_chunks_result = mock.MagicMock(chunks=[]) mock_extract = mock.AsyncMock(return_value=no_chunks_result) - with mock.patch("kreuzberg.extract_file", mock_extract): + with mock.patch("xberg.extract_file", mock_extract): result = await ingest_document(isolated_env / "s.xml", "s.xml", "xml") assert result == [] -class TestChunkViaKreuzberg: +class TestChunkViaXberg: def test_empty_returns_empty(self): from lilbee.data.chunk import chunk_text @@ -2348,7 +2348,7 @@ def _mock_concepts_available(self): yield @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_concept_extraction_called_during_ingest( self, mock_extract_file, isolated_env, mock_svc @@ -2365,7 +2365,7 @@ async def test_concept_extraction_called_during_ingest( mock_svc.concepts.extract_concepts_batch.assert_called() @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_concept_disabled_skips_extraction( self, mock_extract_file, isolated_env, mock_svc @@ -2380,7 +2380,7 @@ async def test_concept_disabled_skips_extraction( mock_svc.concepts.extract_concepts_batch.assert_not_called() @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_concept_failure_does_not_break_ingest( self, mock_extract_file, isolated_env, mock_svc @@ -2397,7 +2397,7 @@ async def test_concept_failure_does_not_break_ingest( assert "concept_test2.txt" in result.added @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_concept_write_failure_does_not_fail_files( self, mock_extract_file, isolated_env, mock_svc @@ -2419,7 +2419,7 @@ async def test_concept_write_failure_does_not_fail_files( assert result.failed == [] @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_cluster_rebuild_called_after_sync( self, mock_extract_file, isolated_env, mock_svc @@ -2436,7 +2436,7 @@ async def test_cluster_rebuild_called_after_sync( mock_svc.concepts.rebuild_clusters.assert_called() @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_cluster_rebuild_failure_does_not_break_sync( self, mock_extract_file, isolated_env, mock_svc @@ -2454,7 +2454,7 @@ async def test_cluster_rebuild_failure_does_not_break_sync( assert "rebuild_test.txt" in result.added @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_graph_none_skips_indexing(self, mock_extract_file, isolated_env, mock_svc): """When get_graph() returns None, concept indexing is skipped gracefully.""" @@ -2468,7 +2468,7 @@ async def test_graph_none_skips_indexing(self, mock_extract_file, isolated_env, assert "graph_none_test.txt" in result.added @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_concepts_unavailable_skips_rebuild( self, mock_extract_file, isolated_env, mock_svc @@ -2485,7 +2485,7 @@ async def test_concepts_unavailable_skips_rebuild( mock_svc.concepts.rebuild_clusters.assert_not_called() @mock.patch( - "kreuzberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_kreuzberg_result() + "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() ) async def test_concepts_unavailable_skips_indexing( self, mock_extract_file, isolated_env, mock_svc @@ -2589,7 +2589,7 @@ async def test_whitespace_pages_yield_no_chunks(self, mock_svc): class TestIngestDocumentOcrPath: - @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) async def test_empty_pdf_warns_no_usable_text(self, mock_kf, isolated_env, mock_svc, caplog): mock_kf.return_value = mock.MagicMock(chunks=[]) from lilbee.data.ingest import ingest_document @@ -2600,15 +2600,15 @@ async def test_empty_pdf_warns_no_usable_text(self, mock_kf, isolated_env, mock_ assert result == [] assert "no usable text" in caplog.text - @mock.patch("kreuzberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) async def test_streams_per_page_progress_ticks(self, mock_kf, isolated_env, mock_svc): import json cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" - result_obj = _make_kreuzberg_result(num_chunks=1, has_pages=True) + result_obj = _make_xberg_result(num_chunks=1, has_pages=True) async def fake_extract(path, *, config): - # Simulate kreuzberg calling the registered backend once per scanned page. + # Simulate xberg calling the registered backend once per scanned page. from lilbee.data.ingest.vision_ocr_backend import ocr_requests token = json.loads(config["ocr"].backend_options)["req"] diff --git a/tests/test_services.py b/tests/test_services.py index 37b3afa1d..c19fcd0a2 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -19,19 +19,19 @@ def isolated_cfg(): class TestSyncVisionOcrBackend: - def _patch_kreuzberg(self, monkeypatch, *, listed): + def _patch_xberg(self, monkeypatch, *, listed): reg = MagicMock() unreg = MagicMock() - monkeypatch.setattr("kreuzberg.list_ocr_backends", lambda: listed) - monkeypatch.setattr("kreuzberg.register_ocr_backend", reg) - monkeypatch.setattr("kreuzberg.unregister_ocr_backend", unreg) + monkeypatch.setattr("xberg.list_ocr_backends", lambda: listed) + monkeypatch.setattr("xberg.register_ocr_backend", reg) + monkeypatch.setattr("xberg.unregister_ocr_backend", unreg) return reg, unreg def test_registers_when_model_set_and_absent(self, monkeypatch): from lilbee.app.services import sync_vision_ocr_backend monkeypatch.setattr(cfg, "vision_model", "vendor/glm-ocr") - reg, unreg = self._patch_kreuzberg(monkeypatch, listed=["tesseract"]) + reg, unreg = self._patch_xberg(monkeypatch, listed=["tesseract"]) sync_vision_ocr_backend(MagicMock()) reg.assert_called_once() unreg.assert_not_called() @@ -40,12 +40,12 @@ def test_rebinds_to_current_provider_when_already_registered(self, monkeypatch): """A rebuilt provider must replace the stale binding: unregister then re-register. ``reset_services`` shuts the old provider down; if sync left the prior - registration in place, kreuzberg would keep routing OCR to the dead provider. + registration in place, xberg would keep routing OCR to the dead provider. """ from lilbee.app.services import sync_vision_ocr_backend monkeypatch.setattr(cfg, "vision_model", "vendor/glm-ocr") - reg, unreg = self._patch_kreuzberg(monkeypatch, listed=["lilbee-vision"]) + reg, unreg = self._patch_xberg(monkeypatch, listed=["lilbee-vision"]) sync_vision_ocr_backend(MagicMock()) unreg.assert_called_once_with("lilbee-vision") reg.assert_called_once() @@ -54,7 +54,7 @@ def test_unregisters_when_model_cleared(self, monkeypatch): from lilbee.app.services import sync_vision_ocr_backend monkeypatch.setattr(cfg, "vision_model", "") - reg, unreg = self._patch_kreuzberg(monkeypatch, listed=["lilbee-vision"]) + reg, unreg = self._patch_xberg(monkeypatch, listed=["lilbee-vision"]) sync_vision_ocr_backend(MagicMock()) unreg.assert_called_once_with("lilbee-vision") reg.assert_not_called() @@ -63,7 +63,7 @@ def test_noop_when_no_model_and_absent(self, monkeypatch): from lilbee.app.services import sync_vision_ocr_backend monkeypatch.setattr(cfg, "vision_model", "") - reg, unreg = self._patch_kreuzberg(monkeypatch, listed=["tesseract"]) + reg, unreg = self._patch_xberg(monkeypatch, listed=["tesseract"]) sync_vision_ocr_backend(MagicMock()) reg.assert_not_called() unreg.assert_not_called() @@ -77,7 +77,7 @@ def test_settings_role_reload_syncs_vision_backend(self, monkeypatch): set_services(make_mock_services()) try: monkeypatch.setattr(cfg, "vision_model", "org/V-GGUF/v-Q4_K_M.gguf") - reg, _unreg = self._patch_kreuzberg(monkeypatch, listed=["tesseract"]) + reg, _unreg = self._patch_xberg(monkeypatch, listed=["tesseract"]) _reload_changed_roles({"vision_model"}) reg.assert_called_once() finally: diff --git a/tests/test_vision_ocr_backend.py b/tests/test_vision_ocr_backend.py index b39df7a08..b68661647 100644 --- a/tests/test_vision_ocr_backend.py +++ b/tests/test_vision_ocr_backend.py @@ -1,4 +1,4 @@ -"""Tests for the lilbee vision OCR backend (kreuzberg custom OCR plugin).""" +"""Tests for the lilbee vision OCR backend (xberg custom OCR plugin).""" from __future__ import annotations @@ -25,7 +25,7 @@ def default_fn(image_bytes, model, prompt, *, timeout): def _cfg(*, vlm_prompt=None, backend_options=None): - """The OcrConfig JSON string kreuzberg hands process_image.""" + """The OcrConfig JSON string xberg hands process_image.""" return json.dumps({"vlm_prompt": vlm_prompt, "backend_options": backend_options}) diff --git a/tools/wheel-build/build_lilbee_binary.sh b/tools/wheel-build/build_lilbee_binary.sh index ab5b9d815..6ffcc7f9d 100755 --- a/tools/wheel-build/build_lilbee_binary.sh +++ b/tools/wheel-build/build_lilbee_binary.sh @@ -76,7 +76,7 @@ uv run --no-sync python -m nuitka \ --include-package=tiktoken --include-package-data=tiktoken \ --include-package=tiktoken_ext --include-package-data=tiktoken_ext \ --include-package-data=numpy \ - --include-package=kreuzberg --include-package-data=kreuzberg \ + --include-package=xberg --include-package-data=xberg \ --include-package=litellm --include-package=litellm.llms --include-package-data=litellm \ --include-package=crawl4ai --include-package-data=crawl4ai \ --include-package=fake_useragent --include-package-data=fake_useragent \ diff --git a/uv.lock b/uv.lock index e8db7d860..6dc5b9257 100644 --- a/uv.lock +++ b/uv.lock @@ -1452,14 +1452,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437 }, ] -[[package]] -name = "kreuzberg" -version = "5.0.0rc32" -source = { path = "../../../../kreuzberg/.worktrees/rc32/dist/kreuzberg-5.0.0rc32-cp310-abi3-macosx_11_0_arm64.whl" } -wheels = [ - { filename = "kreuzberg-5.0.0rc32-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:79477dcd3e77e8401baa82620cb51ae500f3c553c3a251e8f204ca64cc76da9d" }, -] - [[package]] name = "lance-namespace" version = "0.5.2" @@ -1603,7 +1595,6 @@ dependencies = [ { name = "httpx" }, { name = "huggingface-hub" }, { name = "jinja2" }, - { name = "kreuzberg" }, { name = "lancedb" }, { name = "lilbee-engine" }, { name = "litestar" }, @@ -1618,6 +1609,7 @@ dependencies = [ { name = "typer" }, { name = "typing-extensions" }, { name = "uvicorn" }, + { name = "xberg" }, ] [package.optional-dependencies] @@ -1669,7 +1661,6 @@ requires-dist = [ { name = "httpx" }, { name = "huggingface-hub", specifier = ">=1.11.0" }, { name = "jinja2", specifier = ">=2.11.3" }, - { name = "kreuzberg", path = "../../../../kreuzberg/.worktrees/rc32/dist/kreuzberg-5.0.0rc32-cp310-abi3-macosx_11_0_arm64.whl" }, { name = "lancedb" }, { name = "lilbee", extras = ["crawler", "litellm", "graph"], marker = "extra == 'release'" }, { name = "lilbee-engine", directory = "packaging/engine-wheel" }, @@ -1690,6 +1681,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "typing-extensions", specifier = ">=4.5.0" }, { name = "uvicorn", specifier = ">=0.30" }, + { name = "xberg", path = "../../../../xberg-alef-trial-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" }, ] provides-extras = ["litellm", "graph", "crawler", "cuda12", "release"] @@ -4374,6 +4366,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/c7/8528ac2dfa2c1e6708f647df7ae144ead13f0a31146f43c7264b4942bf12/wrapt-2.1.2-py3-none-any.whl", hash = "sha256:b8fd6fa2b2c4e7621808f8c62e8317f4aae56e59721ad933bac5239d913cf0e8", size = 43993 }, ] +[[package]] +name = "xberg" +version = "1.0.0rc1" +source = { path = "../../../../xberg-alef-trial-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" } +wheels = [ + { filename = "xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:b8b30e44015999d52cf802a5aa8f7e54d89c87422797dc024402ee2a9cf8e015" }, +] + [[package]] name = "xxhash" version = "3.6.0" From 1cc9a0086715c695008973eecef6ee7654176572 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:52:45 -0400 Subject: [PATCH 04/77] feat(wiki): adopt xberg footnote/citation API, gut hand-rolled verification (bb-548) xberg ships the footnote/citation API (xberg-io/xberg#649) built for lilbee's wiki use case. Replace lilbee's own excerpt verification and unmarked-claim scanning with xberg.verify_excerpt / xberg.find_unmarked_claims, dropping the lowercasing _normalize, the _is_content_line helper, and the CITE_RE/INFERENCE_RE regex usage. Verification is now case-sensitive (xberg's contract) instead of case-insensitive; citations.py's check was already case-sensitive so only the wiki/citation.py path changes behavior. Domain wrappers (CitationRecord, render_citation_block, chunk page/line resolution) stay. --- src/lilbee/wiki/citation.py | 41 +++++++++--------------------------- src/lilbee/wiki/citations.py | 6 ++++-- tests/test_citation.py | 7 ++++-- tests/test_cli.py | 4 +++- 4 files changed, 22 insertions(+), 36 deletions(-) diff --git a/src/lilbee/wiki/citation.py b/src/lilbee/wiki/citation.py index 2ef63b9ce..e64159e10 100644 --- a/src/lilbee/wiki/citation.py +++ b/src/lilbee/wiki/citation.py @@ -10,9 +10,7 @@ from lilbee.wiki.grammar import ( CITATION_BLOCK_COMMENT, CITATION_BLOCK_SEP, - CITE_RE, FOOTNOTE_RE, - INFERENCE_RE, ) @@ -80,29 +78,24 @@ def verify_citation(citation: CitationRecord, source_text: str) -> CitationStatu by comparing ``citation.source_hash`` against the current file hash and checking file presence. """ + from xberg import verify_excerpt + if not citation["excerpt"]: return CitationStatus.EXCERPT_MISSING - if _normalize(citation["excerpt"]) in _normalize(source_text): + if verify_excerpt(citation["excerpt"], source_text): return CitationStatus.VALID return CitationStatus.EXCERPT_MISSING def find_unmarked_claims(markdown: str) -> list[str]: - """Find statements that are neither cited ``[^srcN]`` nor marked ``[*inference*]``. - Scans non-empty, non-metadata lines in the body (before the citation block). - Returns the text of each unmarked line. + """Find body statements that are neither cited ``[^srcN]`` nor marked ``[*inference*]``. + + Delegates to xberg's footnote/citation API over the body (frontmatter and the + citation block stripped). """ - body = extract_body(markdown) - lines = body.splitlines() - unmarked: list[str] = [] - for line in lines: - stripped = line.strip() - if not _is_content_line(stripped): - continue - if CITE_RE.search(stripped) or INFERENCE_RE.search(stripped): - continue - unmarked.append(stripped) - return unmarked + from xberg import find_unmarked_claims as _find_unmarked_claims + + return _find_unmarked_claims(extract_body(markdown)) def strip_citation_block(markdown: str) -> str: @@ -154,15 +147,6 @@ def _strip_frontmatter(markdown: str) -> str: return markdown -def _is_content_line(stripped: str) -> bool: - """Return True if a line contains a substantive claim (not heading/blank/marker).""" - if not stripped: - return False - if stripped.startswith("#"): - return False - return stripped != CITATION_BLOCK_SEP - - def _format_source_ref(rec: CitationRecord) -> str: """Format a CitationRecord into a human-readable footnote reference.""" ref = rec["source_filename"] @@ -180,8 +164,3 @@ def _format_source_ref(rec: CitationRecord) -> str: if rec["excerpt"]: ref += f', excerpt: "{rec["excerpt"]}"' return ref - - -def _normalize(text: str) -> str: - """Normalize whitespace for fuzzy excerpt matching.""" - return " ".join(text.split()).lower() diff --git a/src/lilbee/wiki/citations.py b/src/lilbee/wiki/citations.py index 579582e15..d826f0630 100644 --- a/src/lilbee/wiki/citations.py +++ b/src/lilbee/wiki/citations.py @@ -152,8 +152,10 @@ def verify_citations( config: Config, ) -> list[CitationRecord]: """Filter citation records, keeping only those whose excerpts are in the chunks.""" + from xberg import verify_excerpt + wiki_prefix = config.wiki_dir + "/" - all_chunk_text = normalize_whitespace(" ".join(c.chunk for c in chunks)) + all_chunk_text = " ".join(c.chunk for c in chunks) verified: list[CitationRecord] = [] for rec in citation_records: if rec["source_filename"].startswith(wiki_prefix): @@ -162,7 +164,7 @@ def verify_citations( if rec["claim_type"] == "inference" or not rec["excerpt"]: verified.append(rec) continue - if normalize_whitespace(rec["excerpt"]) in all_chunk_text: + if verify_excerpt(rec["excerpt"], all_chunk_text): verified.append(rec) else: log.debug("Citation %s excerpt not found in %s, dropping", rec["citation_key"], label) diff --git a/tests/test_citation.py b/tests/test_citation.py index bb9131bb4..0f5bc3884 100644 --- a/tests/test_citation.py +++ b/tests/test_citation.py @@ -201,9 +201,12 @@ def test_whitespace_normalized_for_matching(self): rec = _citation_record(excerpt="gradual\n typing") assert verify_citation(rec, "supports gradual typing here") == CitationStatus.VALID - def test_case_insensitive_matching(self): + def test_case_sensitive_matching(self): + # Verification is case-sensitive (xberg.verify_excerpt, adopted via bb-548): + # a case mismatch fails, an exact-case (whitespace-normalized) match passes. rec = _citation_record(excerpt="Gradual Typing") - assert verify_citation(rec, "gradual typing module") == CitationStatus.VALID + assert verify_citation(rec, "gradual typing module") == CitationStatus.EXCERPT_MISSING + assert verify_citation(rec, "uses Gradual Typing here") == CitationStatus.VALID class TestFindUnmarkedClaims: diff --git a/tests/test_cli.py b/tests/test_cli.py index 959279333..2a0fb0e5e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3056,13 +3056,15 @@ def test_status_with_pages(self, mock_svc, isolated_env): cfg.wiki = True cfg.wiki_dir = "wiki" (isolated_env / "wiki" / "summaries").mkdir(parents=True) - (isolated_env / "wiki" / "summaries" / "a.md").write_text("content") + # An unmarked claim (a sentence with no citation) -> lint flags a warning. + (isolated_env / "wiki" / "summaries" / "a.md").write_text("Python is a typed language.\n") (isolated_env / "wiki" / "drafts").mkdir(parents=True) (isolated_env / "wiki" / "drafts" / "b.md").write_text("content") mock_svc.store.get_citations_for_wiki.return_value = [] result = runner.invoke(app, ["wiki", "status"]) assert result.exit_code == 0 assert "1" in result.output # summaries count + assert "warning(s)" in result.output # the unmarked claim is linted def test_status_json_output(self, mock_svc, isolated_env): cfg.wiki = True From cef45cc4cc790eebe854a3348714aefb80642a9a Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Thu, 25 Jun 2026 14:45:56 -0400 Subject: [PATCH 05/77] feat: route semantic-chunking embeddings through lilbee's fleet (bb-548 Surface 1) Semantic chunking previously had xberg download its own ONNX preset embedder for topic-boundary detection. Expose the fleet's OpenAI-compatible embeddings endpoint via a new LLMProvider.embedding_endpoint() (FleetProvider returns the placed EMBED replica's /v1 base + bare model id; RoutingProvider delegates; SDK returns None), and build the semantic EmbeddingConfig with EmbeddingModelType.llm() pointed at it, falling back to the preset when no fleet endpoint exists. Verified end to end: a doc ingested with semantic chunking embeds through the fleet and is searchable. Also make lilbee's package __getattr__ resolve real submodules (e.g. lilbee.providers) instead of only Lilbee, so attribute-based access (mock.patch string targets) works without a prior import; unknown names still raise AttributeError. --- src/lilbee/__init__.py | 10 ++++- src/lilbee/data/chunk.py | 50 +++++++++++++++++++---- src/lilbee/providers/base.py | 15 +++++++ src/lilbee/providers/fleet/client.py | 10 +++++ src/lilbee/providers/fleet/provider.py | 20 +++++++++- src/lilbee/providers/routing_provider.py | 5 +++ src/lilbee/providers/sdk_llm_provider.py | 7 ++++ tests/providers/test_sdk_llm_provider.py | 7 ++++ tests/test_chunker.py | 51 +++++++++++++++++++++++- tests/test_fleet_client.py | 6 +++ tests/test_fleet_provider.py | 18 +++++++++ tests/test_ingest.py | 2 + tests/test_providers.py | 22 ++++++++++ 13 files changed, 211 insertions(+), 12 deletions(-) diff --git a/src/lilbee/__init__.py b/src/lilbee/__init__.py index 8de176784..4a09c6fa6 100644 --- a/src/lilbee/__init__.py +++ b/src/lilbee/__init__.py @@ -120,4 +120,12 @@ def __getattr__(name: str) -> object: from lilbee.api import Lilbee return Lilbee - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + # Resolve real submodules on attribute access (e.g. ``lilbee.providers`` from a + # mock.patch string target) without forcing eager imports at package load. + # Unknown names still raise AttributeError. + import importlib + + try: + return importlib.import_module(f"{__name__}.{name}") + except ModuleNotFoundError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None diff --git a/src/lilbee/data/chunk.py b/src/lilbee/data/chunk.py index ff11e1b6c..b3d48bff9 100644 --- a/src/lilbee/data/chunk.py +++ b/src/lilbee/data/chunk.py @@ -8,7 +8,9 @@ from lilbee.core.config import cfg if TYPE_CHECKING: - from xberg import ChunkingConfig + from xberg import ChunkingConfig, EmbeddingConfig + + from lilbee.providers.base import EmbeddingEndpoint CHARS_PER_TOKEN = 4 @@ -36,21 +38,53 @@ def _show_download_progress() -> bool: return os.environ.get(_DISABLE_PROGRESS_ENV, "0").lower() not in ("1", "true") +def _semantic_embedding_endpoint() -> EmbeddingEndpoint | None: + """lilbee's own embeddings endpoint for semantic-chunk boundary detection, so + xberg reuses the fleet instead of downloading a preset. None -> use the preset + (no fleet endpoint, e.g. a remote provider).""" + from lilbee.app.services import get_services + + return get_services().provider.embedding_endpoint() + + +def _semantic_embedding_config() -> EmbeddingConfig: + """EmbeddingConfig for semantic chunking: route boundary-detection embeddings + at lilbee's fleet via the Llm variant when available, else xberg's 'fast' preset. + """ + from xberg import EmbeddingConfig, EmbeddingModelType + + # The public xberg.LlmConfig is a distinct class from the one EmbeddingModelType.llm() + # accepts; the constructor's isinstance check only passes the internal type (alef-m07). + from xberg._xberg import LlmConfig + + endpoint = _semantic_embedding_endpoint() + # xberg's .pyi omits the per-variant constructors alef #147 added at runtime; + # .llm()/.preset() work but aren't declared in the stub yet (alef-m07). + if endpoint is not None: + # endpoint.model is the bare id the endpoint routes by (no provider prefix); + # endpoint.base_url already carries the /v1 the client appends /embeddings to. + model = EmbeddingModelType.llm( # type: ignore[attr-defined] + LlmConfig( + model=endpoint.model, + base_url=endpoint.base_url, + api_key=endpoint.api_key, + ) + ) + else: + model = EmbeddingModelType.preset(_SEMANTIC_EMBEDDING_PRESET) # type: ignore[attr-defined] + return EmbeddingConfig(model=model, show_download_progress=_show_download_progress()) + + def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: """Build an xberg ChunkingConfig from the current cfg.""" - from xberg import ChunkingConfig, EmbeddingConfig, EmbeddingModelType + from xberg import ChunkingConfig max_chars, max_overlap = _char_budget() if use_semantic and cfg.semantic_chunking: return ChunkingConfig( chunker_type=_SEMANTIC_CHUNKER, - embedding=EmbeddingConfig( - # xberg's .pyi omits the per-variant constructors alef #147 added at - # runtime; preset() works but isn't declared in the stub yet. - model=EmbeddingModelType.preset(_SEMANTIC_EMBEDDING_PRESET), # type: ignore[attr-defined] - show_download_progress=_show_download_progress(), - ), + embedding=_semantic_embedding_config(), topic_threshold=cfg.topic_threshold, max_characters=max_chars, overlap=max_overlap, diff --git a/src/lilbee/providers/base.py b/src/lilbee/providers/base.py index 916406645..d1aa43724 100644 --- a/src/lilbee/providers/base.py +++ b/src/lilbee/providers/base.py @@ -223,6 +223,16 @@ class StreamFinish: last, when the backend reports them).""" +@dataclass(frozen=True) +class EmbeddingEndpoint: + """OpenAI-compatible embeddings endpoint, for routing third-party embedders + (e.g. xberg's semantic chunker) at lilbee's own fleet instead of a download.""" + + base_url: str + model: str + api_key: str + + class LLMProvider(Protocol): """Protocol for pluggable LLM backends.""" @@ -230,6 +240,11 @@ def embed(self, texts: list[str]) -> list[list[float]]: """Embed a batch of texts, return list of vectors.""" ... + def embedding_endpoint(self) -> EmbeddingEndpoint | None: + """The OpenAI-compatible embeddings endpoint to hand to a third-party + embedder, or None when there isn't a routable one (caller falls back).""" + ... + @overload def chat( self, diff --git a/src/lilbee/providers/fleet/client.py b/src/lilbee/providers/fleet/client.py index 8b2a4664e..c40c54af8 100644 --- a/src/lilbee/providers/fleet/client.py +++ b/src/lilbee/providers/fleet/client.py @@ -334,6 +334,16 @@ def __init__( # Monotonic stamp of the last mark_unhealthy; consulted only while unhealthy. self._unhealthy_since = 0.0 + @property + def base_url(self) -> str: + """The llama-swap endpoint this client posts to.""" + return self._base + + @property + def model(self) -> str: + """The replica model id llama-swap routes this client's requests by.""" + return self._model + @property def healthy(self) -> bool: """Routable: healthy, or unhealthy past the ``_UNHEALTHY_RETRY_S`` cool-down.""" diff --git a/src/lilbee/providers/fleet/provider.py b/src/lilbee/providers/fleet/provider.py index bafbd3415..85b562ecb 100644 --- a/src/lilbee/providers/fleet/provider.py +++ b/src/lilbee/providers/fleet/provider.py @@ -22,7 +22,7 @@ from lilbee.core.config import cfg from lilbee.modelhub.registry import ModelRegistry -from lilbee.providers.base import ProviderError, ProviderErrorKind +from lilbee.providers.base import EmbeddingEndpoint, ProviderError, ProviderErrorKind from lilbee.providers.fleet import planning from lilbee.providers.fleet.client import LlamaServerClient, is_connection_failure from lilbee.providers.fleet.swap_config import cold_load_timeout_s @@ -626,6 +626,24 @@ def embed(self, texts: list[str]) -> list[list[float]]: clients = self._require_clients(WorkerRole.EMBED) return _call_with_failover(clients, lambda client: client.embed(texts)) + def embedding_endpoint(self) -> EmbeddingEndpoint | None: + """The EMBED role's OpenAI endpoint, so a third-party embedder (xberg's + semantic chunker) reuses the fleet instead of downloading a model. Places + the role on demand; returns None if the fleet can't serve embeddings. + """ + try: + clients = self._require_clients(WorkerRole.EMBED) + except ProviderError: + return None + client = clients[0] + # The caller's HTTP client appends /embeddings, so base_url must carry the + # /v1 prefix; the model is the bare replica id llama-swap routes by (a + # provider-prefixed name 404s). llama-server ignores the bearer token but + # OpenAI clients still require one. + return EmbeddingEndpoint( + base_url=f"{client.base_url}/v1", model=client.model, api_key="lilbee-local" + ) + def vision_ocr( self, png_bytes: bytes, model: str, prompt: str = "", *, timeout: float | None = None ) -> str: diff --git a/src/lilbee/providers/routing_provider.py b/src/lilbee/providers/routing_provider.py index 0e4537524..424981202 100644 --- a/src/lilbee/providers/routing_provider.py +++ b/src/lilbee/providers/routing_provider.py @@ -16,6 +16,7 @@ ChatStreamItem, ChatToolResult, ClosableIterator, + EmbeddingEndpoint, LLMProvider, ProviderError, ) @@ -82,6 +83,10 @@ def embed(self, texts: list[str]) -> list[list[float]]: ref = parse_model_ref(cfg.embedding_model) return self._pick_backend(ref).embed(texts) + def embedding_endpoint(self) -> EmbeddingEndpoint | None: + ref = parse_model_ref(cfg.embedding_model) + return self._pick_backend(ref).embedding_endpoint() + @overload def chat( self, diff --git a/src/lilbee/providers/sdk_llm_provider.py b/src/lilbee/providers/sdk_llm_provider.py index 3ad01f719..3d59b9d63 100644 --- a/src/lilbee/providers/sdk_llm_provider.py +++ b/src/lilbee/providers/sdk_llm_provider.py @@ -26,6 +26,7 @@ ChatStreamItem, ChatToolResult, ClosableIterator, + EmbeddingEndpoint, FinishReason, LLMProvider, ProviderError, @@ -103,6 +104,12 @@ def _ensure_initialized(self) -> None: inject_provider_keys() self._initialized = True + def embedding_endpoint(self) -> EmbeddingEndpoint | None: + # Remote SDK backends: keep a third-party embedder (xberg's semantic + # chunker) on its own preset rather than routing it through the user's + # API credentials/quota. + return None + def embed(self, texts: list[str]) -> list[list[float]]: """Embed texts via the configured backend.""" self._ensure_initialized() diff --git a/tests/providers/test_sdk_llm_provider.py b/tests/providers/test_sdk_llm_provider.py index 346972319..55b813311 100644 --- a/tests/providers/test_sdk_llm_provider.py +++ b/tests/providers/test_sdk_llm_provider.py @@ -731,3 +731,10 @@ def test_idempotent_and_preserves_backend_state(self) -> None: provider.shutdown() assert backend.complete_calls == [] assert backend.embed_calls == [] + + +class TestEmbeddingEndpoint: + def test_returns_none(self) -> None: + # Remote SDK backends expose no fleet endpoint to route a third-party + # embedder through; semantic chunking falls back to its own preset. + assert SdkLLMProvider(FakeBackend()).embedding_endpoint() is None diff --git a/tests/test_chunker.py b/tests/test_chunker.py index 8dcb534c3..02716ed8a 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -13,6 +13,32 @@ from lilbee.data.chunk import chunk_text +class _FakeProvider: + def __init__(self, endpoint): + self._endpoint = endpoint + + def embedding_endpoint(self): + return self._endpoint + + +class _FakeServices: + def __init__(self, endpoint): + self.provider = _FakeProvider(endpoint) + + +def _patch_embedding_endpoint(monkeypatch, endpoint): + """Point chunk._semantic_embedding_endpoint's get_services() at a fake provider.""" + monkeypatch.setattr("lilbee.app.services.get_services", lambda: _FakeServices(endpoint)) + + +@pytest.fixture(autouse=True) +def _no_fleet_embedding_endpoint(monkeypatch): + """No live fleet in unit tests: the provider reports no embeddings endpoint, so + semantic chunking falls back to xberg's preset embedder (and the real + _semantic_embedding_endpoint runs instead of placing the fleet's embed role).""" + _patch_embedding_endpoint(monkeypatch, None) + + @dataclass class _FakeMeta: """Stand-in for tree-sitter ChunkContext.metadata in chunk_code tests.""" @@ -127,11 +153,32 @@ def test_semantic_respects_max_chars_when_embedding_present(self, monkeypatch): result = build_chunking_config() assert result.max_characters == 512 * CHARS_PER_TOKEN assert result.embedding is not None - # Built via the real EmbeddingModelType.preset() constructor (xberg/alef #147), - # not the bare-string shorthand. + # No fleet endpoint (the autouse fixture) -> the bundled preset embedder, + # built via the real EmbeddingModelType.preset() constructor (xberg/alef #147). assert result.embedding.model.type == "preset" assert str(result.embedding.model) == '{"type":"preset","name":"fast"}' + def test_semantic_routes_embeddings_through_fleet(self, monkeypatch): + """With a fleet embeddings endpoint, semantic chunking embeds via the Llm + variant (bb-548 Surface 1) rather than downloading the preset.""" + from lilbee.core.config import cfg + from lilbee.data.chunk import build_chunking_config + from lilbee.providers.base import EmbeddingEndpoint + + monkeypatch.setattr(cfg, "semantic_chunking", True) + _patch_embedding_endpoint( + monkeypatch, + EmbeddingEndpoint( + base_url="http://127.0.0.1:9001", model="nomic-embed", api_key="lilbee-local" + ), + ) + result = build_chunking_config() + assert result.embedding is not None + assert result.embedding.model.type == "llm" + # The Llm variant carries lilbee's fleet endpoint, so xberg POSTs there. + assert "127.0.0.1:9001" in str(result.embedding.model) + assert "nomic-embed" in str(result.embedding.model) + def test_char_budget_when_disabled(self, monkeypatch): from lilbee.core.config import cfg from lilbee.data.chunk import CHARS_PER_TOKEN, build_chunking_config diff --git a/tests/test_fleet_client.py b/tests/test_fleet_client.py index bf62ee943..278e5ca19 100644 --- a/tests/test_fleet_client.py +++ b/tests/test_fleet_client.py @@ -53,6 +53,12 @@ def _client(handler=_handler) -> LlamaServerClient: return client +def test_base_url_and_model_properties_expose_endpoint() -> None: + client = _unprobed_client() + assert client.base_url == "http://gpu0" + assert client.model == "test-model" + + def test_rerank_scores_pairs_via_rank_pooling() -> None: seen: dict[str, list] = {} diff --git a/tests/test_fleet_provider.py b/tests/test_fleet_provider.py index 9295b572d..a2d671617 100644 --- a/tests/test_fleet_provider.py +++ b/tests/test_fleet_provider.py @@ -151,6 +151,24 @@ def test_embed_routes_to_least_busy_replica() -> None: busy.embed.assert_not_called() +def test_embedding_endpoint_exposes_embed_replica() -> None: + client = _fake_client() + client.base_url = "http://127.0.0.1:9001" + client.model = "embed-x" + p = _provider_with_clients({WorkerRole.EMBED: [client]}) + ep = p.embedding_endpoint() + assert ep is not None + # base_url carries /v1 (the HTTP client appends /embeddings); model is the bare id. + assert ep.base_url == "http://127.0.0.1:9001/v1" + assert ep.model == "embed-x" + assert ep.api_key # OpenAI clients require a (ignored) bearer token + + +def test_embedding_endpoint_none_when_embed_absent() -> None: + p = _provider_with_clients({}) + assert p.embedding_endpoint() is None + + def test_adopt_swap_builds_a_client_per_replica(monkeypatch) -> None: launches = [_fake_launch(WorkerRole.EMBED), _fake_launch(WorkerRole.EMBED)] _install_engine(monkeypatch, launches=launches) diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 21b5ebece..79877aa52 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -1985,6 +1985,8 @@ def test_topic_threshold_propagates_to_every_mode(self, monkeypatch): monkeypatch.setattr(cfg, "semantic_chunking", True) monkeypatch.setattr(cfg, "topic_threshold", 0.42) + # No live fleet in unit tests: semantic chunking falls back to the preset. + monkeypatch.setattr("lilbee.data.chunk._semantic_embedding_endpoint", lambda: None) for mode in ExtractMode: config = extraction_config(mode) assert config["chunking"].chunker_type == "semantic" diff --git a/tests/test_providers.py b/tests/test_providers.py index 730e4ce64..f2ebc08cb 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -249,6 +249,28 @@ def test_routes_chat_to_litellm_for_api_model(self) -> None: kwargs = mock_litellm.chat.call_args.kwargs assert kwargs["stream"] is False + def test_embedding_endpoint_delegates_to_local_for_native_model(self, monkeypatch) -> None: + from lilbee.core.config import cfg + + rp = self._make_provider() + sentinel = object() + mock_local = mock.MagicMock() + mock_local.embedding_endpoint.return_value = sentinel + rp._local = mock_local + monkeypatch.setattr(cfg, "embedding_model", "org/repo/embed.gguf") + assert rp.embedding_endpoint() is sentinel + + def test_embedding_endpoint_delegates_to_sdk_for_remote_model(self, monkeypatch) -> None: + from lilbee.core.config import cfg + + rp = self._make_provider() + mock_sdk = mock.MagicMock() + mock_sdk.embedding_endpoint.return_value = None + rp._sdk_provider = mock_sdk + monkeypatch.setattr(cfg, "embedding_model", "openai/text-embedding-3-small") + assert rp.embedding_endpoint() is None + mock_sdk.embedding_endpoint.assert_called_once() + def test_supports_tools_delegates_to_sdk_backend_for_remote_ref(self) -> None: rp = self._make_provider() mock_sdk = mock.MagicMock() From 719c705a141e32ea89db1bafcd60f9849cf57814 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Thu, 25 Jun 2026 18:00:24 -0400 Subject: [PATCH 06/77] docs: surface multi-language OCR (LILBEE_OCR_LANGUAGE) Scanned-document OCR isn't English-only: Tesseract covers 100+ languages via cfg.ocr_language (env LILBEE_OCR_LANGUAGE, e.g. eng+deu, verified on a German scan) and vision models read many scripts. Note it in the README feature line and supported-formats table, the marketing site's scans/OCR card, and the usage OCR comparison. ('Ask in plain English' is unchanged: it describes phrasing a query, not a content-language limit.) --- README.md | 4 ++-- docs/usage.md | 1 + site/index.html | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 848b3dbf7..1c7995284 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ lilbee splits indexing by what's being read: - **Prose and structured documents** (PDFs, Office files, ebooks, HTML, 90+ formats) go through [Xberg] with heading-aware chunking, so each chunk keeps its section context. - **Code** goes through [tree-sitter]'s AST-aware splitter across [150+ languages](https://github.com/Goldziher/tree-sitter-language-pack), so chunks map to functions, classes, and modules instead of arbitrary line ranges. -- **Scanned PDFs and photos** go through OCR: Tesseract for plain text, or a local / remote vision model that keeps tables and layout as markdown. +- **Scanned PDFs and photos** go through OCR in 100+ languages: Tesseract for plain text (set `LILBEE_OCR_LANGUAGE`, e.g. `eng+deu`), or a local / remote vision model that keeps tables and layout as markdown. Retrieval returns things that make sense on their own, not fragments cut through an argument or a function signature. @@ -363,7 +363,7 @@ Document extraction powered by [Xberg], code chunking by [tree-sitter]. lilbee h | Format | Extensions | Requires | | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | PDF | `.pdf` | none | -| Scanned PDF | `.pdf` (no extractable text) | [Tesseract](https://github.com/tesseract-ocr/tesseract) (auto, plain text), or a GGUF vision model via the native mtmd backend (recommended, preserves tables, headings, and layout as markdown) | +| Scanned PDF | `.pdf` (no extractable text) | [Tesseract](https://github.com/tesseract-ocr/tesseract) (auto, plain text, 100+ languages via `LILBEE_OCR_LANGUAGE`), or a GGUF vision model via the native mtmd backend (recommended, preserves tables, headings, and layout as markdown) | | Office | `.docx`, `.xlsx`, `.pptx`, `.doc`, `.xls`, `.ppt`, `.odt`, `.ods`, `.rtf` | none | | eBook | `.epub`, `.fb2` | none | | Images (OCR) | `.png`, `.jpg`, `.jpeg`, `.tiff`, `.bmp`, `.webp`, `.gif`, `.heic`, `.avif`, `.jp2`/`.j2k`/`.jpx` (JPEG-2000) | [Tesseract](https://github.com/tesseract-ocr/tesseract) or a vision model | diff --git a/docs/usage.md b/docs/usage.md index 0df505d5f..17c77bd84 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1063,6 +1063,7 @@ vision model is configured, it takes precedence. | | Tesseract | Vision model | |---|---|---| | **Output** | Plain text | Structured markdown (tables, headings) | +| **Languages** | 100+ via `LILBEE_OCR_LANGUAGE` (e.g. `eng+deu`) | Many scripts (model-dependent) | | **Retrieval quality** | Fragments lose context | Chunks preserve semantic boundaries | | **Install** | System package (`brew`/`apt`) | Native GGUF via the built-in mtmd backend, or any vision model reachable via the SDK backend (`pip install --pre 'lilbee[litellm]'` / `uv tool install --prerelease=allow 'lilbee[litellm]'`) | | **Best for** | Simple text-only scans | Tables, multi-column layouts, formatted docs | diff --git a/site/index.html b/site/index.html index 67710be67..6fd750a3e 100644 --- a/site/index.html +++ b/site/index.html @@ -507,7 +507,7 @@

crawl websites for offline search

scans & OCR

-

Old scans and photos go through OCR or a local vision model and come out as searchable markdown, layout intact.

+

Old scans and photos go through OCR in 100+ languages, or a local vision model, and come out as searchable markdown, layout intact.

remembers what you tell it

From 9166f512dbc62d7a9fbd47f59c4a95b314b7164f Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:15:57 -0400 Subject: [PATCH 07/77] fix(ocr): round-trip ocr_language through the persist path; document the +-delimited format The OCR language setting is a list, but the writable surfaces (lilbee set, the TUI settings screen, the HTTP and MCP config APIs) persist list values to config.toml newline-joined. The validator only split on '+' and ',', so a multi-language value like eng+deu was written as "eng\ndeu" and reloaded as the single bogus token ['eng\ndeu'], which Tesseract rejects. Split on newlines too, matching the crawl list-field validators. Document the format consistently across surfaces: '+' is the canonical join (Tesseract's own convention), commas also accepted. README, the OCR docs, the settings help text, and the field comment now say so, and the docs show the env, CLI, config, TUI, and HTTP/MCP entry points. --- docs/usage.md | 13 +++++++++++++ src/lilbee/app/settings_map.py | 2 +- src/lilbee/core/config/model.py | 16 +++++++++++----- tests/test_config.py | 6 ++++++ 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 17c77bd84..551a3f86b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1080,6 +1080,19 @@ brew install tesseract # macOS sudo apt install tesseract-ocr # Ubuntu/Debian ``` +By default Tesseract reads English (`eng`). To OCR other languages, set the +language codes; xberg downloads the Tesseract data on first use. Join multiple +languages with `+` (Tesseract's own convention), for example `eng+deu` for +English and German. The setting is the same value everywhere: + +```bash +export LILBEE_OCR_LANGUAGE=eng+deu # environment +lilbee set ocr_language eng+deu # CLI / persisted to config.toml +``` + +You can also set it in the `/settings` screen in the TUI, or over the HTTP and +MCP settings APIs. Commas work in place of `+` if you prefer (`eng,deu`). + ### Vision models lilbee runs vision OCR in one of two ways: diff --git a/src/lilbee/app/settings_map.py b/src/lilbee/app/settings_map.py index 83348c50e..e94abe661 100644 --- a/src/lilbee/app/settings_map.py +++ b/src/lilbee/app/settings_map.py @@ -725,7 +725,7 @@ def get_default(key: str) -> object: list, nullable=False, group=SettingGroup.INGEST, - help_text="Tesseract OCR language codes when no vision model is set (e.g. eng, eng+deu)", + help_text="Tesseract OCR languages when no vision model is set; '+'-join, e.g. eng+deu", ), "worker_pool_eager_start": SettingDef( bool, diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index 45bd6d56b..4fc7ee2aa 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -685,12 +685,18 @@ def _parse_enable_ocr(cls, v: Any) -> bool | None: @field_validator("ocr_language", mode="before") @classmethod def _parse_ocr_language(cls, v: Any) -> list[str]: - """Accept a list or a comma/plus-separated string; never return empty. - - Tesseract uses ``+`` to join languages and xberg errors on an empty - list, so blank input falls back to English. + """Accept a list or a ``+``/comma/newline-separated string; never empty. + + Tesseract joins languages with ``+`` (e.g. ``eng+deu``), so that is the + canonical user-facing form. Commas are also accepted. Newlines are + accepted because ``app.settings`` joins list values with ``\\n`` when it + persists them to config.toml; without splitting on it a multi-language + value would reload as one malformed token. Blank input falls back to + English, since xberg errors on an empty list. """ - items = v.replace("+", ",").split(",") if isinstance(v, str) else (v or []) + if isinstance(v, str): + v = v.replace("+", ",").replace("\n", ",").split(",") + items = v or [] langs = [s.strip() for s in items if isinstance(s, str) and s.strip()] return langs or ["eng"] diff --git a/tests/test_config.py b/tests/test_config.py index 254b3beca..fc8ff9cdc 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -164,6 +164,12 @@ def test_env_comma_separated(self, tmp_path): def test_direct_list(self): assert Config(ocr_language=["spa"]).ocr_language == ["spa"] + def test_persisted_newline_form_round_trips(self): + """app.settings joins list values with '\\n' before writing config.toml; + the validator must split on it so a multi-language value reloads intact.""" + persisted = "\n".join(["eng", "deu"]) + assert Config(ocr_language=persisted).ocr_language == ["eng", "deu"] + def test_empty_list_falls_back_to_english(self): assert Config(ocr_language=[]).ocr_language == ["eng"] From 0cac97193db766b07964a6e4a202582110582f8d Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:20:34 -0400 Subject: [PATCH 08/77] chore(lint): clear pre-existing lint gate flags Annotate the two EmbeddingModelType.llm()/.preset() type-ignores with the sanctioned style-check: allow-smell tag. They cover xberg constructors that exist at runtime but are missing from the .pyi stub (alef #147), not owned-attribute smells, so the diff-scoped gate's default advice does not apply. Also reformat test_ingest.py to the locked ruff so format-check is clean. --- src/lilbee/data/chunk.py | 6 ++++-- tests/test_ingest.py | 12 +++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/lilbee/data/chunk.py b/src/lilbee/data/chunk.py index b3d48bff9..262388933 100644 --- a/src/lilbee/data/chunk.py +++ b/src/lilbee/data/chunk.py @@ -63,7 +63,7 @@ def _semantic_embedding_config() -> EmbeddingConfig: if endpoint is not None: # endpoint.model is the bare id the endpoint routes by (no provider prefix); # endpoint.base_url already carries the /v1 the client appends /embeddings to. - model = EmbeddingModelType.llm( # type: ignore[attr-defined] + model = EmbeddingModelType.llm( # type: ignore[attr-defined] # style-check: allow-smell LlmConfig( model=endpoint.model, base_url=endpoint.base_url, @@ -71,7 +71,9 @@ def _semantic_embedding_config() -> EmbeddingConfig: ) ) else: - model = EmbeddingModelType.preset(_SEMANTIC_EMBEDDING_PRESET) # type: ignore[attr-defined] + model = EmbeddingModelType.preset( # type: ignore[attr-defined] # style-check: allow-smell + _SEMANTIC_EMBEDDING_PRESET + ) return EmbeddingConfig(model=model, show_download_progress=_show_download_progress()) diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 79877aa52..ea0ed0e88 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -176,9 +176,7 @@ def _make_empty_result(): return result -@mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() -) +@mock.patch("xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result()) class TestSync: async def test_empty_documents_dir(self, mock_extract_file, isolated_env): from lilbee.data.ingest import SyncResult, sync @@ -683,9 +681,7 @@ async def _fail(path, name, ct, **kwargs): assert "qflaky.txt" not in result.updated -@mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() -) +@mock.patch("xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result()) class TestSyncCancellation: """Tests for cancel support and atomic per-file delete in sync.""" @@ -2043,9 +2039,7 @@ def test_classify(self, filename, expected): assert classify_file(Path(filename)) == expected -@mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() -) +@mock.patch("xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result()) class TestSyncStructuredFormats: async def test_xml_file_ingested( self, From 49e99101cedfaf69ed7f5ac0d8a091d9f861da39 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:48:30 -0400 Subject: [PATCH 09/77] refactor(chunk): route semantic-chunk embeddings through lilbee's plugin backend Replace the EmbeddingModelType.llm fleet-endpoint wiring with xberg's plugin embedding backend, the same mechanism the vision OCR backend already uses. xberg's semantic chunker now calls back into lilbee's embedder (provider.embed) to detect topic boundaries, so the model that vectorizes chunks for retrieval is the one that decides where they split -- in every case, not just when a fleet endpoint exists. xberg's bundled ONNX preset is no longer used. LilbeeEmbeddingBackend (name/initialize/dimensions/embed/shutdown) is registered in app.services.sync_embedding_backend, mirroring sync_vision_ocr_backend: it reads cfg.embedding_dim live and routes through provider.embed (so an embedding-model swap needs no re-registration), and re-binds whenever the provider is rebuilt. The retrieval embedding pipeline (Embedder, provider.embed, the store) is untouched. Drops the now-unused EmbeddingEndpoint dataclass, the embedding_endpoint protocol method and its three implementations, and the fleet client's base_url/model accessors (net -136 lines). --- src/lilbee/app/services.py | 28 +++++++++ src/lilbee/data/chunk.py | 61 ++++--------------- src/lilbee/data/embedding_backend.py | 50 +++++++++++++++ src/lilbee/data/ingest/types.py | 8 +++ src/lilbee/providers/base.py | 15 ----- src/lilbee/providers/fleet/client.py | 10 --- src/lilbee/providers/fleet/provider.py | 20 +----- src/lilbee/providers/routing_provider.py | 5 -- src/lilbee/providers/sdk_llm_provider.py | 7 --- tests/providers/test_sdk_llm_provider.py | 7 --- tests/test_chunker.py | 77 ++---------------------- tests/test_fleet_client.py | 6 -- tests/test_fleet_provider.py | 18 ------ tests/test_ingest.py | 2 - tests/test_providers.py | 22 ------- tests/test_services.py | 46 ++++++++++++++ 16 files changed, 148 insertions(+), 234 deletions(-) create mode 100644 src/lilbee/data/embedding_backend.py diff --git a/src/lilbee/app/services.py b/src/lilbee/app/services.py index 95de70821..cd87fd45e 100644 --- a/src/lilbee/app/services.py +++ b/src/lilbee/app/services.py @@ -186,6 +186,7 @@ def get_services() -> Services: # vision. Set ``cfg.worker_pool_eager_start = false`` for headless scripts # where mount time matters more than first-call latency. sync_vision_ocr_backend(provider) + sync_embedding_backend(provider) # Eager start is the default: pay the spawn cost per role server at TUI mount if cfg.worker_pool_eager_start: from contextlib import suppress @@ -222,6 +223,33 @@ def sync_vision_ocr_backend(provider: LLMProvider) -> None: unregister_ocr_backend(OcrBackendName.LILBEE_VISION) +def sync_embedding_backend(provider: LLMProvider) -> None: + """Register lilbee's embedder as xberg's plugin embedding backend. + + The semantic chunker uses it to detect topic boundaries with the same model + that vectorizes chunks, instead of xberg's bundled ONNX preset. The backend + reads ``cfg.embedding_dim`` live and routes through ``provider.embed`` (which + resolves the embedding model per call), so a model swap needs no + re-registration -- but it captures ``provider.embed``, so it must re-bind + whenever the provider is rebuilt (``reset_services``). + """ + from xberg import ( + list_embedding_backends, + register_embedding_backend, + unregister_embedding_backend, + ) + + from lilbee.core.config import cfg + from lilbee.data.embedding_backend import LilbeeEmbeddingBackend + from lilbee.data.ingest.types import EmbeddingBackendName + + if EmbeddingBackendName.LILBEE in list_embedding_backends(): + unregister_embedding_backend(EmbeddingBackendName.LILBEE) + register_embedding_backend( + LilbeeEmbeddingBackend(embed_fn=provider.embed, dim_fn=lambda: cfg.embedding_dim) + ) + + def set_services(services: Services | None) -> None: """Replace the cached Services singleton (for testing).""" global _svc diff --git a/src/lilbee/data/chunk.py b/src/lilbee/data/chunk.py index 262388933..81ccb0d38 100644 --- a/src/lilbee/data/chunk.py +++ b/src/lilbee/data/chunk.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os from typing import TYPE_CHECKING from lilbee.core.config import cfg @@ -10,15 +9,10 @@ if TYPE_CHECKING: from xberg import ChunkingConfig, EmbeddingConfig - from lilbee.providers.base import EmbeddingEndpoint - CHARS_PER_TOKEN = 4 _SEMANTIC_CHUNKER = "semantic" _MARKDOWN_CHUNKER = "markdown" -# Xberg silently falls back to a non-semantic path when embedding is None. -_SEMANTIC_EMBEDDING_PRESET = "fast" -_DISABLE_PROGRESS_ENV = "HF_HUB_DISABLE_PROGRESS_BARS" def _char_budget() -> tuple[int, int]: @@ -28,53 +22,20 @@ def _char_budget() -> tuple[int, int]: return max_chars, max_overlap -def _show_download_progress() -> bool: - """Honor lilbee's global progress-bar suppression (set for quiet/JSON modes). - - lilbee defaults ``HF_HUB_DISABLE_PROGRESS_BARS`` on in ``__init__``; mirroring - it here keeps the embedding-model download silent instead of hardcoding a bar - that would corrupt JSON output. - """ - return os.environ.get(_DISABLE_PROGRESS_ENV, "0").lower() not in ("1", "true") - - -def _semantic_embedding_endpoint() -> EmbeddingEndpoint | None: - """lilbee's own embeddings endpoint for semantic-chunk boundary detection, so - xberg reuses the fleet instead of downloading a preset. None -> use the preset - (no fleet endpoint, e.g. a remote provider).""" - from lilbee.app.services import get_services - - return get_services().provider.embedding_endpoint() - - def _semantic_embedding_config() -> EmbeddingConfig: - """EmbeddingConfig for semantic chunking: route boundary-detection embeddings - at lilbee's fleet via the Llm variant when available, else xberg's 'fast' preset. - """ + """EmbeddingConfig for semantic chunking. Boundary-detection embeddings route to + lilbee's embedder, registered as xberg's plugin backend in + ``app.services.sync_embedding_backend``, so the model that vectorizes chunks for + retrieval is the one that decides where they split.""" from xberg import EmbeddingConfig, EmbeddingModelType - # The public xberg.LlmConfig is a distinct class from the one EmbeddingModelType.llm() - # accepts; the constructor's isinstance check only passes the internal type (alef-m07). - from xberg._xberg import LlmConfig - - endpoint = _semantic_embedding_endpoint() - # xberg's .pyi omits the per-variant constructors alef #147 added at runtime; - # .llm()/.preset() work but aren't declared in the stub yet (alef-m07). - if endpoint is not None: - # endpoint.model is the bare id the endpoint routes by (no provider prefix); - # endpoint.base_url already carries the /v1 the client appends /embeddings to. - model = EmbeddingModelType.llm( # type: ignore[attr-defined] # style-check: allow-smell - LlmConfig( - model=endpoint.model, - base_url=endpoint.base_url, - api_key=endpoint.api_key, - ) - ) - else: - model = EmbeddingModelType.preset( # type: ignore[attr-defined] # style-check: allow-smell - _SEMANTIC_EMBEDDING_PRESET - ) - return EmbeddingConfig(model=model, show_download_progress=_show_download_progress()) + # Lazy: importing ingest.types at module scope cycles back through chunk.py. + from lilbee.data.ingest.types import EmbeddingBackendName + + # xberg's .pyi omits the per-variant constructors (alef #147); .plugin() works at runtime. + name = EmbeddingBackendName.LILBEE + model = EmbeddingModelType.plugin(name) # type: ignore[attr-defined] # style-check: allow-smell + return EmbeddingConfig(model=model) def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: diff --git a/src/lilbee/data/embedding_backend.py b/src/lilbee/data/embedding_backend.py new file mode 100644 index 000000000..ff7f1f7ab --- /dev/null +++ b/src/lilbee/data/embedding_backend.py @@ -0,0 +1,50 @@ +"""lilbee's embedder exposed as a xberg plugin embedding backend. + +xberg's semantic chunker needs embeddings to detect topic boundaries. Registering +this backend routes those embeddings to lilbee's own embedder (whatever model the +fleet serves), so the model that vectorizes chunks for retrieval is the same one +that decides where chunks split. Without it, xberg falls back to its bundled ONNX +preset, a different model. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from lilbee.data.ingest.types import EmbeddingBackendName + +if TYPE_CHECKING: + from collections.abc import Callable + + +class LilbeeEmbeddingBackend: + """Routes xberg's boundary-detection embeddings to lilbee's embedder. + + The functional path is injected: ``embed_fn`` is the provider's batch embed + (read live, so an embedding-model swap needs no re-registration) and + ``dim_fn`` reports the current vector width. xberg calls ``initialize`` once + at registration, then ``dimensions`` and ``embed`` while chunking. Methods are + sync; xberg invokes them on its own worker threads. + """ + + def __init__( + self, + *, + embed_fn: Callable[[list[str]], list[list[float]]], + dim_fn: Callable[[], int], + ) -> None: + self._embed_fn = embed_fn + self._dim_fn = dim_fn + + def name(self) -> str: + return EmbeddingBackendName.LILBEE + + def initialize(self) -> None: ... + + def shutdown(self) -> None: ... + + def dimensions(self) -> int: + return self._dim_fn() + + def embed(self, texts: list[str]) -> list[list[float]]: + return self._embed_fn(texts) diff --git a/src/lilbee/data/ingest/types.py b/src/lilbee/data/ingest/types.py index f2bc7e5ea..babdf36f1 100644 --- a/src/lilbee/data/ingest/types.py +++ b/src/lilbee/data/ingest/types.py @@ -54,6 +54,14 @@ class OcrBackendName(StrEnum): LILBEE_VISION = "lilbee-vision" +class EmbeddingBackendName(StrEnum): + """Embedding backends registered with xberg. lilbee registers its own embedder + as a plugin so the semantic chunker detects boundaries with the same model that + vectorizes chunks.""" + + LILBEE = "lilbee" + + class ExtractMode(StrEnum): """Extraction topology: paginated (PDFs/images) vs markdown output (text formats).""" diff --git a/src/lilbee/providers/base.py b/src/lilbee/providers/base.py index d1aa43724..916406645 100644 --- a/src/lilbee/providers/base.py +++ b/src/lilbee/providers/base.py @@ -223,16 +223,6 @@ class StreamFinish: last, when the backend reports them).""" -@dataclass(frozen=True) -class EmbeddingEndpoint: - """OpenAI-compatible embeddings endpoint, for routing third-party embedders - (e.g. xberg's semantic chunker) at lilbee's own fleet instead of a download.""" - - base_url: str - model: str - api_key: str - - class LLMProvider(Protocol): """Protocol for pluggable LLM backends.""" @@ -240,11 +230,6 @@ def embed(self, texts: list[str]) -> list[list[float]]: """Embed a batch of texts, return list of vectors.""" ... - def embedding_endpoint(self) -> EmbeddingEndpoint | None: - """The OpenAI-compatible embeddings endpoint to hand to a third-party - embedder, or None when there isn't a routable one (caller falls back).""" - ... - @overload def chat( self, diff --git a/src/lilbee/providers/fleet/client.py b/src/lilbee/providers/fleet/client.py index c40c54af8..8b2a4664e 100644 --- a/src/lilbee/providers/fleet/client.py +++ b/src/lilbee/providers/fleet/client.py @@ -334,16 +334,6 @@ def __init__( # Monotonic stamp of the last mark_unhealthy; consulted only while unhealthy. self._unhealthy_since = 0.0 - @property - def base_url(self) -> str: - """The llama-swap endpoint this client posts to.""" - return self._base - - @property - def model(self) -> str: - """The replica model id llama-swap routes this client's requests by.""" - return self._model - @property def healthy(self) -> bool: """Routable: healthy, or unhealthy past the ``_UNHEALTHY_RETRY_S`` cool-down.""" diff --git a/src/lilbee/providers/fleet/provider.py b/src/lilbee/providers/fleet/provider.py index 85b562ecb..bafbd3415 100644 --- a/src/lilbee/providers/fleet/provider.py +++ b/src/lilbee/providers/fleet/provider.py @@ -22,7 +22,7 @@ from lilbee.core.config import cfg from lilbee.modelhub.registry import ModelRegistry -from lilbee.providers.base import EmbeddingEndpoint, ProviderError, ProviderErrorKind +from lilbee.providers.base import ProviderError, ProviderErrorKind from lilbee.providers.fleet import planning from lilbee.providers.fleet.client import LlamaServerClient, is_connection_failure from lilbee.providers.fleet.swap_config import cold_load_timeout_s @@ -626,24 +626,6 @@ def embed(self, texts: list[str]) -> list[list[float]]: clients = self._require_clients(WorkerRole.EMBED) return _call_with_failover(clients, lambda client: client.embed(texts)) - def embedding_endpoint(self) -> EmbeddingEndpoint | None: - """The EMBED role's OpenAI endpoint, so a third-party embedder (xberg's - semantic chunker) reuses the fleet instead of downloading a model. Places - the role on demand; returns None if the fleet can't serve embeddings. - """ - try: - clients = self._require_clients(WorkerRole.EMBED) - except ProviderError: - return None - client = clients[0] - # The caller's HTTP client appends /embeddings, so base_url must carry the - # /v1 prefix; the model is the bare replica id llama-swap routes by (a - # provider-prefixed name 404s). llama-server ignores the bearer token but - # OpenAI clients still require one. - return EmbeddingEndpoint( - base_url=f"{client.base_url}/v1", model=client.model, api_key="lilbee-local" - ) - def vision_ocr( self, png_bytes: bytes, model: str, prompt: str = "", *, timeout: float | None = None ) -> str: diff --git a/src/lilbee/providers/routing_provider.py b/src/lilbee/providers/routing_provider.py index 424981202..0e4537524 100644 --- a/src/lilbee/providers/routing_provider.py +++ b/src/lilbee/providers/routing_provider.py @@ -16,7 +16,6 @@ ChatStreamItem, ChatToolResult, ClosableIterator, - EmbeddingEndpoint, LLMProvider, ProviderError, ) @@ -83,10 +82,6 @@ def embed(self, texts: list[str]) -> list[list[float]]: ref = parse_model_ref(cfg.embedding_model) return self._pick_backend(ref).embed(texts) - def embedding_endpoint(self) -> EmbeddingEndpoint | None: - ref = parse_model_ref(cfg.embedding_model) - return self._pick_backend(ref).embedding_endpoint() - @overload def chat( self, diff --git a/src/lilbee/providers/sdk_llm_provider.py b/src/lilbee/providers/sdk_llm_provider.py index 3d59b9d63..3ad01f719 100644 --- a/src/lilbee/providers/sdk_llm_provider.py +++ b/src/lilbee/providers/sdk_llm_provider.py @@ -26,7 +26,6 @@ ChatStreamItem, ChatToolResult, ClosableIterator, - EmbeddingEndpoint, FinishReason, LLMProvider, ProviderError, @@ -104,12 +103,6 @@ def _ensure_initialized(self) -> None: inject_provider_keys() self._initialized = True - def embedding_endpoint(self) -> EmbeddingEndpoint | None: - # Remote SDK backends: keep a third-party embedder (xberg's semantic - # chunker) on its own preset rather than routing it through the user's - # API credentials/quota. - return None - def embed(self, texts: list[str]) -> list[list[float]]: """Embed texts via the configured backend.""" self._ensure_initialized() diff --git a/tests/providers/test_sdk_llm_provider.py b/tests/providers/test_sdk_llm_provider.py index 55b813311..346972319 100644 --- a/tests/providers/test_sdk_llm_provider.py +++ b/tests/providers/test_sdk_llm_provider.py @@ -731,10 +731,3 @@ def test_idempotent_and_preserves_backend_state(self) -> None: provider.shutdown() assert backend.complete_calls == [] assert backend.embed_calls == [] - - -class TestEmbeddingEndpoint: - def test_returns_none(self) -> None: - # Remote SDK backends expose no fleet endpoint to route a third-party - # embedder through; semantic chunking falls back to its own preset. - assert SdkLLMProvider(FakeBackend()).embedding_endpoint() is None diff --git a/tests/test_chunker.py b/tests/test_chunker.py index 02716ed8a..e54a95d9a 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -13,32 +13,6 @@ from lilbee.data.chunk import chunk_text -class _FakeProvider: - def __init__(self, endpoint): - self._endpoint = endpoint - - def embedding_endpoint(self): - return self._endpoint - - -class _FakeServices: - def __init__(self, endpoint): - self.provider = _FakeProvider(endpoint) - - -def _patch_embedding_endpoint(monkeypatch, endpoint): - """Point chunk._semantic_embedding_endpoint's get_services() at a fake provider.""" - monkeypatch.setattr("lilbee.app.services.get_services", lambda: _FakeServices(endpoint)) - - -@pytest.fixture(autouse=True) -def _no_fleet_embedding_endpoint(monkeypatch): - """No live fleet in unit tests: the provider reports no embeddings endpoint, so - semantic chunking falls back to xberg's preset embedder (and the real - _semantic_embedding_endpoint runs instead of placing the fleet's embed role).""" - _patch_embedding_endpoint(monkeypatch, None) - - @dataclass class _FakeMeta: """Stand-in for tree-sitter ChunkContext.metadata in chunk_code tests.""" @@ -153,31 +127,10 @@ def test_semantic_respects_max_chars_when_embedding_present(self, monkeypatch): result = build_chunking_config() assert result.max_characters == 512 * CHARS_PER_TOKEN assert result.embedding is not None - # No fleet endpoint (the autouse fixture) -> the bundled preset embedder, - # built via the real EmbeddingModelType.preset() constructor (xberg/alef #147). - assert result.embedding.model.type == "preset" - assert str(result.embedding.model) == '{"type":"preset","name":"fast"}' - - def test_semantic_routes_embeddings_through_fleet(self, monkeypatch): - """With a fleet embeddings endpoint, semantic chunking embeds via the Llm - variant (bb-548 Surface 1) rather than downloading the preset.""" - from lilbee.core.config import cfg - from lilbee.data.chunk import build_chunking_config - from lilbee.providers.base import EmbeddingEndpoint - - monkeypatch.setattr(cfg, "semantic_chunking", True) - _patch_embedding_endpoint( - monkeypatch, - EmbeddingEndpoint( - base_url="http://127.0.0.1:9001", model="nomic-embed", api_key="lilbee-local" - ), - ) - result = build_chunking_config() - assert result.embedding is not None - assert result.embedding.model.type == "llm" - # The Llm variant carries lilbee's fleet endpoint, so xberg POSTs there. - assert "127.0.0.1:9001" in str(result.embedding.model) - assert "nomic-embed" in str(result.embedding.model) + # Boundary detection routes to lilbee's embedder via xberg's plugin backend + # (registered in app.services.sync_embedding_backend), not the ONNX preset. + assert result.embedding.model.type == "plugin" + assert str(result.embedding.model) == '{"type":"plugin","name":"lilbee"}' def test_char_budget_when_disabled(self, monkeypatch): from lilbee.core.config import cfg @@ -201,28 +154,6 @@ def test_disabled_does_not_attach_embedding(self, monkeypatch): result = build_chunking_config() assert result.embedding is None - def test_download_progress_off_when_globally_suppressed(self, monkeypatch): - """quiet/JSON modes suppress HF progress bars; the embedding config mirrors that.""" - from lilbee.data.chunk import _show_download_progress - - monkeypatch.setenv("HF_HUB_DISABLE_PROGRESS_BARS", "1") - assert _show_download_progress() is False - - def test_download_progress_on_when_not_suppressed(self, monkeypatch): - from lilbee.data.chunk import _show_download_progress - - monkeypatch.setenv("HF_HUB_DISABLE_PROGRESS_BARS", "0") - assert _show_download_progress() is True - - def test_download_progress_default_off_when_unset(self, monkeypatch): - """lilbee defaults the env var on at import, so the bar stays off by default.""" - from lilbee.data.chunk import _show_download_progress - - monkeypatch.delenv("HF_HUB_DISABLE_PROGRESS_BARS", raising=False) - # Unset env reads as "not disabled" -> progress allowed; lilbee's __init__ - # sets it to "1" in real runs, so this documents the bare-helper contract. - assert _show_download_progress() is True - def test_heading_path_shares_char_budget(self, monkeypatch): """The heading-aware path uses the same token->char budget as the default path.""" from lilbee.core.config import cfg diff --git a/tests/test_fleet_client.py b/tests/test_fleet_client.py index 278e5ca19..bf62ee943 100644 --- a/tests/test_fleet_client.py +++ b/tests/test_fleet_client.py @@ -53,12 +53,6 @@ def _client(handler=_handler) -> LlamaServerClient: return client -def test_base_url_and_model_properties_expose_endpoint() -> None: - client = _unprobed_client() - assert client.base_url == "http://gpu0" - assert client.model == "test-model" - - def test_rerank_scores_pairs_via_rank_pooling() -> None: seen: dict[str, list] = {} diff --git a/tests/test_fleet_provider.py b/tests/test_fleet_provider.py index a2d671617..9295b572d 100644 --- a/tests/test_fleet_provider.py +++ b/tests/test_fleet_provider.py @@ -151,24 +151,6 @@ def test_embed_routes_to_least_busy_replica() -> None: busy.embed.assert_not_called() -def test_embedding_endpoint_exposes_embed_replica() -> None: - client = _fake_client() - client.base_url = "http://127.0.0.1:9001" - client.model = "embed-x" - p = _provider_with_clients({WorkerRole.EMBED: [client]}) - ep = p.embedding_endpoint() - assert ep is not None - # base_url carries /v1 (the HTTP client appends /embeddings); model is the bare id. - assert ep.base_url == "http://127.0.0.1:9001/v1" - assert ep.model == "embed-x" - assert ep.api_key # OpenAI clients require a (ignored) bearer token - - -def test_embedding_endpoint_none_when_embed_absent() -> None: - p = _provider_with_clients({}) - assert p.embedding_endpoint() is None - - def test_adopt_swap_builds_a_client_per_replica(monkeypatch) -> None: launches = [_fake_launch(WorkerRole.EMBED), _fake_launch(WorkerRole.EMBED)] _install_engine(monkeypatch, launches=launches) diff --git a/tests/test_ingest.py b/tests/test_ingest.py index ea0ed0e88..50e7a4b4d 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -1981,8 +1981,6 @@ def test_topic_threshold_propagates_to_every_mode(self, monkeypatch): monkeypatch.setattr(cfg, "semantic_chunking", True) monkeypatch.setattr(cfg, "topic_threshold", 0.42) - # No live fleet in unit tests: semantic chunking falls back to the preset. - monkeypatch.setattr("lilbee.data.chunk._semantic_embedding_endpoint", lambda: None) for mode in ExtractMode: config = extraction_config(mode) assert config["chunking"].chunker_type == "semantic" diff --git a/tests/test_providers.py b/tests/test_providers.py index f2ebc08cb..730e4ce64 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -249,28 +249,6 @@ def test_routes_chat_to_litellm_for_api_model(self) -> None: kwargs = mock_litellm.chat.call_args.kwargs assert kwargs["stream"] is False - def test_embedding_endpoint_delegates_to_local_for_native_model(self, monkeypatch) -> None: - from lilbee.core.config import cfg - - rp = self._make_provider() - sentinel = object() - mock_local = mock.MagicMock() - mock_local.embedding_endpoint.return_value = sentinel - rp._local = mock_local - monkeypatch.setattr(cfg, "embedding_model", "org/repo/embed.gguf") - assert rp.embedding_endpoint() is sentinel - - def test_embedding_endpoint_delegates_to_sdk_for_remote_model(self, monkeypatch) -> None: - from lilbee.core.config import cfg - - rp = self._make_provider() - mock_sdk = mock.MagicMock() - mock_sdk.embedding_endpoint.return_value = None - rp._sdk_provider = mock_sdk - monkeypatch.setattr(cfg, "embedding_model", "openai/text-embedding-3-small") - assert rp.embedding_endpoint() is None - mock_sdk.embedding_endpoint.assert_called_once() - def test_supports_tools_delegates_to_sdk_backend_for_remote_ref(self) -> None: rp = self._make_provider() mock_sdk = mock.MagicMock() diff --git a/tests/test_services.py b/tests/test_services.py index c19fcd0a2..a8874dad1 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -84,6 +84,52 @@ def test_settings_role_reload_syncs_vision_backend(self, monkeypatch): set_services(None) +class TestSyncEmbeddingBackend: + def _patch_xberg(self, monkeypatch, *, listed): + reg = MagicMock() + unreg = MagicMock() + monkeypatch.setattr("xberg.list_embedding_backends", lambda: listed) + monkeypatch.setattr("xberg.register_embedding_backend", reg) + monkeypatch.setattr("xberg.unregister_embedding_backend", unreg) + return reg, unreg + + def test_registers_when_absent(self, monkeypatch): + from lilbee.app.services import sync_embedding_backend + + reg, unreg = self._patch_xberg(monkeypatch, listed=[]) + sync_embedding_backend(MagicMock()) + reg.assert_called_once() + unreg.assert_not_called() + + def test_rebinds_to_current_provider_when_already_registered(self, monkeypatch): + """A rebuilt provider must replace the stale binding: unregister then re-register, + else xberg keeps embedding through the shut-down provider after reset_services.""" + from lilbee.app.services import sync_embedding_backend + + reg, unreg = self._patch_xberg(monkeypatch, listed=["lilbee"]) + sync_embedding_backend(MagicMock()) + unreg.assert_called_once_with("lilbee") + reg.assert_called_once() + + def test_registered_backend_routes_to_provider_embed(self, monkeypatch): + """The registered backend must call provider.embed and report cfg.embedding_dim.""" + from lilbee.app.services import sync_embedding_backend + + reg, _unreg = self._patch_xberg(monkeypatch, listed=[]) + monkeypatch.setattr(cfg, "embedding_dim", 7) + provider = MagicMock() + provider.embed.return_value = [[0.0] * 7] + sync_embedding_backend(provider) + backend = reg.call_args.args[0] + assert backend.name() == "lilbee" + assert backend.dimensions() == 7 + assert backend.embed(["hello"]) == [[0.0] * 7] + provider.embed.assert_called_once_with(["hello"]) + # Lifecycle hooks are no-ops but must exist (xberg calls initialize). + assert backend.initialize() is None + assert backend.shutdown() is None + + class TestServicesDataclass: def test_fields_are_immutable(self): from lilbee.app.services import CrawlerSyncState, Services From d646561552bc5dbf869e69a273df22ad33bb71ab Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:03:12 -0400 Subject: [PATCH 10/77] build: integrate xberg all-PRs wheel; drop plugin type-ignore Point the local xberg source at the all-PRs build. It declares the EmbeddingModelType.plugin/llm/preset/custom constructors in the stub (the alef #147 gap), so the attr-defined ignore and its allow-smell tag are gone. One arg-type ignore remains: the public EmbeddingConfig still types model as the legacy str|int|LlmConfig alias rather than the EmbeddingModelType class its own .plugin() returns (upstream stub inconsistency; the constructor takes the class instance at runtime). --- pyproject.toml | 2 +- src/lilbee/data/chunk.py | 9 +++++---- uv.lock | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6cf18fed0..be805d238 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,7 @@ packages = ["src/lilbee"] lilbee-engine = { path = "packaging/engine-wheel" } # TODO: xberg 1.0.0rc1 is a pre-publication trial wheel; resolve it from the local # build until it lands on PyPI, then drop this source and lock from the index. -xberg = { path = "/Users/tobias/projects/xberg-alef-trial-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" } +xberg = { path = "/Users/tobias/projects/xberg-all-prs-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" } [dependency-groups] dev = [ diff --git a/src/lilbee/data/chunk.py b/src/lilbee/data/chunk.py index 81ccb0d38..a6ba14582 100644 --- a/src/lilbee/data/chunk.py +++ b/src/lilbee/data/chunk.py @@ -32,10 +32,11 @@ def _semantic_embedding_config() -> EmbeddingConfig: # Lazy: importing ingest.types at module scope cycles back through chunk.py. from lilbee.data.ingest.types import EmbeddingBackendName - # xberg's .pyi omits the per-variant constructors (alef #147); .plugin() works at runtime. - name = EmbeddingBackendName.LILBEE - model = EmbeddingModelType.plugin(name) # type: ignore[attr-defined] # style-check: allow-smell - return EmbeddingConfig(model=model) + model = EmbeddingModelType.plugin(EmbeddingBackendName.LILBEE) + # xberg's public EmbeddingConfig still types `model` as the legacy + # str|int|LlmConfig alias, not the EmbeddingModelType class its own .plugin() + # returns; the constructor accepts the class instance at runtime. + return EmbeddingConfig(model=model) # type: ignore[arg-type] def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: diff --git a/uv.lock b/uv.lock index 6dc5b9257..8a586a832 100644 --- a/uv.lock +++ b/uv.lock @@ -1681,7 +1681,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "typing-extensions", specifier = ">=4.5.0" }, { name = "uvicorn", specifier = ">=0.30" }, - { name = "xberg", path = "../../../../xberg-alef-trial-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" }, + { name = "xberg", path = "../../../../xberg-all-prs-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" }, ] provides-extras = ["litellm", "graph", "crawler", "cuda12", "release"] @@ -4369,9 +4369,9 @@ wheels = [ [[package]] name = "xberg" version = "1.0.0rc1" -source = { path = "../../../../xberg-alef-trial-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" } +source = { path = "../../../../xberg-all-prs-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:b8b30e44015999d52cf802a5aa8f7e54d89c87422797dc024402ee2a9cf8e015" }, + { filename = "xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:da563ff860a17ef74ed66f25a6d45e97de533dc578883258ef995138f89a1635" }, ] [[package]] From abb3d1dd9664131355db3cea0b08b4dfc12d3354 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:20:02 -0400 Subject: [PATCH 11/77] test(placement): cover the placement-screen error/guard branches The merge pulled in the multi-GPU placement screen (#438 from feat/local-model-api), whose tests covered the worker error paths but not the _spec_from_editor guard branches, the unknown-button no-op, the clear single-flight guard, or the clear worker's error path (16 lines). Add error-path tests so the merged tree holds the 100% gate. --- tests/test_tui_placement.py | 108 ++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/test_tui_placement.py b/tests/test_tui_placement.py index 05d53236f..89d2fc647 100644 --- a/tests/test_tui_placement.py +++ b/tests/test_tui_placement.py @@ -391,3 +391,111 @@ def _boom(): await pilot.pause() assert any("probe failed" in n for n in notes) + + +@pytest.mark.asyncio +async def test_remove_one_of_multiple_devices(monkeypatch): + """Toggling off a device a role still shares with others removes just that one.""" + from lilbee.cli.tui.screens import placement as screen_mod + + monkeypatch.setattr(screen_mod, "get_placement", lambda: _make_view()) + + app = PlacementTestApp() + async with app.run_test(size=(140, 44)) as pilot: + await pilot.pause() + await pilot.click("#dev-embed-2") # embed -> [0, 2] + await pilot.pause() + assert '"embed": {"devices": [0, 2]}' in _generated(app) + btn = app.screen.query_one("#dev-embed-2", Button) + app.screen.on_button_pressed(Button.Pressed(btn)) # remove CUDA2 (len > 1) -> [0] + await pilot.pause() + assert '"embed": {"devices": [0]}' in _generated(app) + + +@pytest.mark.asyncio +async def test_unknown_button_is_ignored(monkeypatch): + """A Button.Pressed whose id is neither dev- nor rep- is a no-op.""" + from lilbee.cli.tui.screens import placement as screen_mod + + monkeypatch.setattr(screen_mod, "get_placement", lambda: _make_view()) + + app = PlacementTestApp() + async with app.run_test(size=(140, 44)) as pilot: + await pilot.pause() + before = _generated(app) + app.screen.on_button_pressed(Button.Pressed(Button("x", id="unrelated"))) + await pilot.pause() + assert _generated(app) == before # handler returned without re-rendering + + +@pytest.mark.asyncio +async def test_role_without_a_device_surfaces_error(monkeypatch): + """A role left with no GPU fails spec building: red in the panel, error on preview/apply.""" + from lilbee.cli.tui.screens import placement as screen_mod + from lilbee.providers.roles import WorkerRole + + monkeypatch.setattr(screen_mod, "get_placement", lambda: _make_view()) + + app = PlacementTestApp() + async with app.run_test(size=(140, 44)) as pilot: + await pilot.pause() + app.screen._edits[WorkerRole.EMBED].devices.clear() # force the empty-devices guard + notes: list[str] = [] + monkeypatch.setattr(app.screen, "notify", lambda msg, **k: notes.append(msg)) + + app.screen._refresh_generated() + await pilot.pause() + assert "needs at least one GPU" in _generated(app) + + app.screen.action_preview() + await pilot.pause() + app.screen.action_apply() + await pilot.pause() + + assert sum("needs at least one GPU" in n for n in notes) >= 2 + + +@pytest.mark.asyncio +async def test_clear_ignored_while_applying(monkeypatch): + """ctrl+x while an apply/clear is in flight is a no-op (single-flight guard).""" + from lilbee.cli.tui.screens import placement as screen_mod + + monkeypatch.setattr(screen_mod, "get_placement", lambda: _make_view()) + calls: list[object] = [] + monkeypatch.setattr( + screen_mod, "set_placement", lambda spec: calls.append(spec) or _make_view() + ) + + app = PlacementTestApp() + async with app.run_test(size=(140, 44)) as pilot: + await pilot.pause() + app.screen.applying = True + app.screen.action_clear() + await pilot.pause() + + assert calls == [] # guard returned before scheduling the clear worker + + +@pytest.mark.asyncio +async def test_clear_error_notifies(monkeypatch): + """A failure inside the clear worker surfaces as a notification, not a crash.""" + from lilbee.cli.tui.screens import placement as screen_mod + from lilbee.providers.fleet.placement_spec import PlacementError + + monkeypatch.setattr(screen_mod, "get_placement", lambda: _make_view()) + + def _boom(spec): # type: ignore[no-untyped-def] + raise PlacementError("clear failed") + + monkeypatch.setattr(screen_mod, "set_placement", _boom) + + notes: list[str] = [] + app = PlacementTestApp() + async with app.run_test(size=(140, 44)) as pilot: + await pilot.pause() + monkeypatch.setattr(app.screen, "notify", lambda msg, **k: notes.append(msg)) + await pilot.press("ctrl+x") + await app.workers.wait_for_complete() + await pilot.pause() + + assert any("clear failed" in n for n in notes) From 7aa9a9c0a302ebfc20dc41bc64a175b68ae0b926 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:50:59 -0400 Subject: [PATCH 12/77] fix(ocr): read the native OcrConfig in the vision backend callback The all-PRs xberg wheel passes the OcrConfig to register_ocr_backend's process_image as a native object (alef typed trait callbacks), not the JSON string the trial wheel sent. The backend still json.loads-ed it, silently got nothing, and dropped the request token -- so per-page OCR timeout and progress ticks were lost on scanned-PDF vision OCR (a regression vs the old pdf_ocr path this branch replaced). Read vlm_prompt and backend_options off the object directly; verified the timeout and progress callback propagate again. Tests now pass a real xberg OcrConfig instead of a hand-built JSON string. --- src/lilbee/data/ingest/vision_ocr_backend.py | 46 +++++++++----------- tests/test_vision_ocr_backend.py | 29 +++++------- 2 files changed, 32 insertions(+), 43 deletions(-) diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py index 3967de7d1..8eda1e056 100644 --- a/src/lilbee/data/ingest/vision_ocr_backend.py +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -8,7 +8,7 @@ from contextlib import contextmanager from dataclasses import dataclass from importlib.metadata import PackageNotFoundError, version -from typing import TYPE_CHECKING, Any, Protocol, cast +from typing import TYPE_CHECKING, Any, Protocol from lilbee.data.ingest.types import MARKDOWN_MIME, OcrBackendName from lilbee.vision import resolve_ocr_prompt @@ -16,6 +16,8 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator + from xberg import OcrConfig + # Token key inside OcrConfig.backend_options JSON. xberg does not propagate # contextvars into process_image (xberg-4w9), so per-request state travels as # a token on the config and is resolved through the registry below. @@ -74,36 +76,30 @@ def backend_options_for(token: str) -> str: return json.dumps({_REQUEST_TOKEN_KEY: token}) -def _config_to_dict(config: str) -> dict[str, Any]: - """Parse the OcrConfig JSON string xberg passes to process_image into a dict.""" - try: - raw: object = json.loads(config) - except (ValueError, TypeError): - return {} - return cast("dict[str, Any]", raw) if isinstance(raw, dict) else {} - - class _OcrConfigView: - """Typed reader over the xberg OcrConfig (normalized to a dict).""" + """Typed reader over the xberg OcrConfig object passed to process_image. + + xberg hands the callback a native OcrConfig (alef typed trait callbacks), so + its fields are read directly as attributes. + """ - def __init__(self, config: dict[str, Any]) -> None: + def __init__(self, config: OcrConfig) -> None: self._config = config @property def vlm_prompt(self) -> str | None: - return self._config.get("vlm_prompt") + return self._config.vlm_prompt @property def request_token(self) -> str | None: - raw = self._config.get("backend_options") - if isinstance(raw, str): - try: - raw = json.loads(raw) - except (ValueError, TypeError): - return None - if not isinstance(raw, dict): + raw = self._config.backend_options + if raw is None: + return None + try: + options = json.loads(raw) + except (ValueError, TypeError): return None - token = raw.get(_REQUEST_TOKEN_KEY) + token = options.get(_REQUEST_TOKEN_KEY) if isinstance(options, dict) else None return token if isinstance(token, str) else None @@ -148,10 +144,10 @@ def shutdown(self) -> None: ... def backend_type(self) -> str: return "custom" - def process_image(self, image_bytes: bytes, config: str) -> dict[str, Any]: - # xberg serializes the OcrConfig to a JSON string for the callback; - # parse before reading fields. - view = _OcrConfigView(_config_to_dict(config)) + def process_image(self, image_bytes: bytes, config: OcrConfig) -> dict[str, Any]: + # xberg passes the OcrConfig as a native object (alef typed trait + # callbacks); read the request token and prompt override off it. + view = _OcrConfigView(config) model = self._model_ref_fn() prompt = view.vlm_prompt or resolve_ocr_prompt(model) ctx = ocr_requests.get(view.request_token) diff --git a/tests/test_vision_ocr_backend.py b/tests/test_vision_ocr_backend.py index b68661647..351098010 100644 --- a/tests/test_vision_ocr_backend.py +++ b/tests/test_vision_ocr_backend.py @@ -2,8 +2,6 @@ from __future__ import annotations -import json - from lilbee.data.ingest.types import MARKDOWN_MIME, OcrBackendName from lilbee.data.ingest.vision_ocr_backend import ( VisionOcrBackend, @@ -25,8 +23,14 @@ def default_fn(image_bytes, model, prompt, *, timeout): def _cfg(*, vlm_prompt=None, backend_options=None): - """The OcrConfig JSON string xberg hands process_image.""" - return json.dumps({"vlm_prompt": vlm_prompt, "backend_options": backend_options}) + """The native xberg OcrConfig object xberg hands process_image.""" + from xberg import OcrConfig + + return OcrConfig( + backend=OcrBackendName.LILBEE_VISION, + vlm_prompt=vlm_prompt, + backend_options=backend_options, + ) class TestProtocol: @@ -106,23 +110,12 @@ def test_no_context_uses_zero_timeout_and_no_tick(self): assert calls[0][3] == 0.0 def test_malformed_backend_options_ignored(self): + # Non-JSON and valid-but-non-object backend_options both resolve to no token. be, calls = _backend() be.process_image(b"PNG", _cfg(backend_options="not-json")) + be.process_image(b"PNG", _cfg(backend_options="123")) assert calls[0][3] == 0.0 - - def test_malformed_config_string_falls_back_to_defaults(self): - # A non-JSON config string yields an empty view: resolved prompt, zero timeout. - be, calls = _backend(model="vendor/glm-ocr") - be.process_image(b"PNG", "}{ not json") - _, _, prompt, timeout = calls[0] - assert prompt == "OCR" - assert timeout == 0.0 - - def test_non_object_json_config_falls_back_to_defaults(self): - # Valid JSON that isn't an object (e.g. a bare number) is treated as empty. - be, calls = _backend(model="vendor/glm-ocr") - be.process_image(b"PNG", "123") - assert calls[0][2] == "OCR" + assert calls[1][3] == 0.0 class TestRegistry: From 563d2994361284bfc991cb5b5f4fa25f87b1e35e Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 27 Jun 2026 14:32:49 -0400 Subject: [PATCH 13/77] license: relicense from Elastic License 2.0 to MIT Replace the ELv2 text with the MIT License and switch every license identifier: pyproject (license = MIT), README badge + section, the marketing site, flake.nix (licenses.mit; drop the now-unneeded unfree allowance since MIT is free), the AUR PKGBUILDs + .SRCINFO, the flatpak metainfo, and the scoop bucket manifest. --- LICENSE | 114 ++++-------------- README.md | 4 +- bucket/lilbee.json | 2 +- flake.nix | 15 +-- packaging/aur/lilbee-cuda/.SRCINFO | 2 +- packaging/aur/lilbee-cuda/PKGBUILD | 2 +- packaging/aur/lilbee/.SRCINFO | 2 +- packaging/aur/lilbee/PKGBUILD | 2 +- .../io.github.tobocop2.lilbee.metainfo.xml | 2 +- pyproject.toml | 2 +- site/index.html | 4 +- 11 files changed, 34 insertions(+), 117 deletions(-) diff --git a/LICENSE b/LICENSE index 05d308877..c42346e91 100644 --- a/LICENSE +++ b/LICENSE @@ -1,93 +1,21 @@ -Elastic License 2.0 (ELv2) - -Copyright 2025-2026 tobocop2 - -Acceptance - -By using the software, you agree to all of the terms and conditions below. - -Copyright License - -The licensor grants you a non-exclusive, royalty-free, worldwide, -non-sublicensable, non-transferable license to use, copy, distribute, make -available, and prepare derivative works of the software, in each case subject to -the limitations and conditions below. - -Limitations - -You may not provide the software to third parties as a hosted or managed -service, where the service provides users with access to any substantial set of -the features or functionality of the software. - -You may not move, change, disable, or circumvent the license key functionality -in the software, and you may not remove or obscure any functionality in the -software that is protected by the license key. - -You may not alter, remove, or obscure any licensing, copyright, or other notices -of the licensor in the software. Any use of the licensor's trademarks is subject -to applicable law. - -Patents - -The licensor grants you a license, under any patent claims the licensor can -license, or becomes able to license, to make, have made, use, sell, offer for -sale, import and have imported the software, in each case subject to the -limitations and conditions in this license. This license does not cover any -patent claims that you cause to be infringed by modifications or additions to the -software. If you or your company make any written claim that the software -infringes or contributes to infringement of any patent, your patent license for -the software granted under these terms ends immediately. If your company makes -such a claim, your patent license ends immediately for work on behalf of your -company. - -Notices - -You must ensure that anyone who gets a copy of any part of the software from you -also gets a copy of these terms. - -If you modify the software, you must include in any modified copies of the -software prominent notices stating that you have modified the software. - -No Other Rights - -These terms do not imply any licenses other than those expressly granted in -these terms. - -Termination - -If you use the software in violation of these terms, such use is not licensed, -and your licenses will automatically terminate. If the licensor provides you with -a notice of your violation, and you cease all violation of this license no later -than 30 days after you receive that notice, your licenses will be reinstated -retroactively. However, if you violate these terms after such reinstatement, any -additional violation of these terms will cause your licenses to terminate -automatically and permanently. - -No Liability - -As far as the law allows, the software comes as is, without any warranty or -condition, and the licensor will not be liable to you for any damages arising out -of these terms or the use or nature of the software, under any kind of legal -claim. - -Definitions - -The licensor is the entity offering these terms, and the software is the -software the licensor makes available under these terms, including any portion -of it. - -you refers to the individual or entity agreeing to these terms. - -your company is any legal entity, sole proprietorship, or other kind of -organization that you work for, plus all organizations that have control over, -are under the control of, or are under common control with that organization. -control means ownership of substantially all the assets of an entity, or the -power to direct its management and policies by vote, contract, or otherwise. -Control can be direct or indirect. - -your licenses are all the licenses granted to you for the software under these -terms. - -use means anything you do with the software requiring one of your licenses. - -trademark means trademarks, service marks, and similar rights. +MIT License + +Copyright (c) 2025-2026 tobocop2 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 1c7995284..bf48088cf 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ lilbee on PyPI Python 3.11+ Platforms - License: Elastic License 2.0 + License: MIT Obsidian community plugin Glama MCP server score

@@ -421,7 +421,7 @@ lilbee is built and maintained by one person. If it is useful to you, you can ch ## License -Elastic License 2.0 (ELv2). See [LICENSE](LICENSE). +MIT. See [LICENSE](LICENSE). [Xberg]: https://github.com/xberg-io/xberg [LanceDB]: https://lancedb.com diff --git a/bucket/lilbee.json b/bucket/lilbee.json index 507a135a6..e0fc45f39 100644 --- a/bucket/lilbee.json +++ b/bucket/lilbee.json @@ -3,7 +3,7 @@ "description": "Run and manage local AI models, then search your files, code, and crawled web pages, with answers that cite the source. One program, no server to run.", "homepage": "https://lilbee.sh/", "license": { - "identifier": "Elastic-2.0", + "identifier": "MIT", "url": "https://github.com/tobocop2/lilbee/blob/main/LICENSE" }, "architecture": { diff --git a/flake.nix b/flake.nix index 3b3797cd9..7dba1a6b7 100644 --- a/flake.nix +++ b/flake.nix @@ -12,23 +12,12 @@ systems = builtins.attrNames sources.systems; forAllSystems = nixpkgs.lib.genAttrs systems; - # nixpkgs flags Elastic 2.0 as unfree; scope the allow to lilbee only. - mkPkgs = - system: - import nixpkgs { - inherit system; - config.allowUnfreePredicate = - pkg: - builtins.elem (nixpkgs.lib.getName pkg) [ - "lilbee" - "lilbee-bin" - ]; - }; + mkPkgs = system: import nixpkgs { inherit system; }; mkMeta = pkgs: { description = "Run and manage local AI models and search your files, code, and crawled web pages, with cited answers"; homepage = "https://github.com/tobocop2/lilbee"; - license = pkgs.lib.licenses.elastic20; + license = pkgs.lib.licenses.mit; mainProgram = "lilbee"; platforms = systems; sourceProvenance = [ pkgs.lib.sourceTypes.binaryNativeCode ]; diff --git a/packaging/aur/lilbee-cuda/.SRCINFO b/packaging/aur/lilbee-cuda/.SRCINFO index adbca6b47..9436ceb83 100644 --- a/packaging/aur/lilbee-cuda/.SRCINFO +++ b/packaging/aur/lilbee-cuda/.SRCINFO @@ -4,7 +4,7 @@ pkgbase = lilbee-cuda pkgrel = 1 url = https://github.com/tobocop2/lilbee arch = x86_64 - license = custom:Elastic-2.0 + license = MIT provides = lilbee conflicts = lilbee options = !strip diff --git a/packaging/aur/lilbee-cuda/PKGBUILD b/packaging/aur/lilbee-cuda/PKGBUILD index bd38fcaf3..b99b36724 100644 --- a/packaging/aur/lilbee-cuda/PKGBUILD +++ b/packaging/aur/lilbee-cuda/PKGBUILD @@ -5,7 +5,7 @@ pkgrel=1 pkgdesc="Run and manage local AI models and search your files, code, and crawled web pages, with cited answers (CUDA build)" arch=('x86_64') url="https://github.com/tobocop2/lilbee" -license=('custom:Elastic-2.0') +license=('MIT') conflicts=('lilbee') provides=('lilbee') options=('!strip' '!debug') diff --git a/packaging/aur/lilbee/.SRCINFO b/packaging/aur/lilbee/.SRCINFO index 9575d53ef..21917c125 100644 --- a/packaging/aur/lilbee/.SRCINFO +++ b/packaging/aur/lilbee/.SRCINFO @@ -4,7 +4,7 @@ pkgbase = lilbee pkgrel = 1 url = https://github.com/tobocop2/lilbee arch = x86_64 - license = custom:Elastic-2.0 + license = MIT replaces = lilbee-bin options = !strip options = !debug diff --git a/packaging/aur/lilbee/PKGBUILD b/packaging/aur/lilbee/PKGBUILD index ede70c0de..fbc844773 100644 --- a/packaging/aur/lilbee/PKGBUILD +++ b/packaging/aur/lilbee/PKGBUILD @@ -5,7 +5,7 @@ pkgrel=1 pkgdesc="Run and manage local AI models and search your files, code, and crawled web pages, with cited answers" arch=('x86_64') url="https://github.com/tobocop2/lilbee" -license=('custom:Elastic-2.0') +license=('MIT') replaces=('lilbee-bin') options=('!strip' '!debug') source_x86_64=("lilbee-${pkgver}::${url}/releases/download/v${pkgver}/lilbee-linux-${CARCH}") diff --git a/packaging/flatpak/io.github.tobocop2.lilbee.metainfo.xml b/packaging/flatpak/io.github.tobocop2.lilbee.metainfo.xml index 5551d5f30..7fd730718 100644 --- a/packaging/flatpak/io.github.tobocop2.lilbee.metainfo.xml +++ b/packaging/flatpak/io.github.tobocop2.lilbee.metainfo.xml @@ -4,7 +4,7 @@ lilbee Local search engine you can talk to CC0-1.0 - Elastic-2.0 + MIT

lilbee runs and manages local AI models, indexes your files and code, diff --git a/pyproject.toml b/pyproject.toml index be805d238..ca37d5d02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "lilbee" version = "0.6.66b504" description = "Run and manage local AI models, then search your files, code, and the web pages you crawl, with answers that cite the source. Everything runs on your machine in one process: per-project libraries, semantic and hybrid search, vision OCR, an auto-built wiki, CLI, TUI, MCP server, REST API, and Python library. No model server, no database server. Works with your own Ollama or LM Studio models too." readme = "README.md" -license = "Elastic-2.0" +license = "MIT" authors = [{ name = "tobocop2", email = "5562156+tobocop2@users.noreply.github.com" }] requires-python = ">=3.11" keywords = [ diff --git a/site/index.html b/site/index.html index 6fd750a3e..07f277787 100644 --- a/site/index.html +++ b/site/index.html @@ -68,7 +68,7 @@

lilbee v0.6.66b
- Elastic License 2.0 + MIT License 100% coverage typed
@@ -567,7 +567,7 @@

built on

|t|o|b|i|a|s|@|l|i|l|b|e|e|.|s|h| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
lilbee  .  Elastic License 2.0
+
lilbee  .  MIT License
From 13e6bd9ec43551341c547ef67d3151612589ef23 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 27 Jun 2026 14:32:49 -0400 Subject: [PATCH 14/77] style: reformat a merged placement test to the locked ruff --- tests/providers/fleet/test_planning_placement_branch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/providers/fleet/test_planning_placement_branch.py b/tests/providers/fleet/test_planning_placement_branch.py index 4a372bbd6..8963bd24a 100644 --- a/tests/providers/fleet/test_planning_placement_branch.py +++ b/tests/providers/fleet/test_planning_placement_branch.py @@ -63,7 +63,9 @@ def counting(_binary): planning, "_server_model_inputs", lambda roles, *, unified_budget=None: ([], {}, 0) ) monkeypatch.setattr( - planning, "_resolve_placement", lambda *a, **k: Placement(instances=(), unplaceable_roles=()) + planning, + "_resolve_placement", + lambda *a, **k: Placement(instances=(), unplaceable_roles=()), ) planning.resolve_placement_plan(None) planning.resolve_placement_plan(None) From 229b663f7d3b9bfcfae15290dc374815d1a0052b Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sun, 28 Jun 2026 23:53:57 -0400 Subject: [PATCH 15/77] wip(xberg): migrate to the new async extract()/ExtractedDocument API Port lilbee's extraction layer to xberg's redesigned API (extract(ExtractInput) -> ExtractionResult.results[ExtractedDocument], async-only): new data/xberg_extract.py bridge, chunk.py/extract.py/vision_ocr_backend.py updated, process_image returns an ExtractedDocument. Text extraction + chunking verified working. BLOCKED on two xberg wheel defects (filed in ~/projects/kreuzberg): kreuzberg-6a2 (SIGBUS in the PDF OCR pipeline aborts the process) and kreuzberg-9n0 (OcrConfig.backend_options stub dict vs runtime str). OCR ingest + the test-suite migration + the full gate are deferred until the wheel is fixed and rebuilt. --- pyproject.toml | 2 +- src/lilbee/data/chunk.py | 10 +-- src/lilbee/data/ingest/extract.py | 36 +++++----- src/lilbee/data/ingest/vision_ocr_backend.py | 36 ++++------ src/lilbee/data/xberg_extract.py | 75 ++++++++++++++++++++ uv.lock | 6 +- 6 files changed, 113 insertions(+), 52 deletions(-) create mode 100644 src/lilbee/data/xberg_extract.py diff --git a/pyproject.toml b/pyproject.toml index ca37d5d02..730d84745 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,7 @@ packages = ["src/lilbee"] lilbee-engine = { path = "packaging/engine-wheel" } # TODO: xberg 1.0.0rc1 is a pre-publication trial wheel; resolve it from the local # build until it lands on PyPI, then drop this source and lock from the index. -xberg = { path = "/Users/tobias/projects/xberg-all-prs-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" } +xberg = { path = "/Users/tobias/projects/kreuzberg-wt-pywheel/dist/xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl" } [dependency-groups] dev = [ diff --git a/src/lilbee/data/chunk.py b/src/lilbee/data/chunk.py index a6ba14582..bbd35973f 100644 --- a/src/lilbee/data/chunk.py +++ b/src/lilbee/data/chunk.py @@ -67,7 +67,9 @@ def chunk_text( if not text or not text.strip(): return [] - from xberg import ChunkingConfig, ExtractionConfig, extract_bytes_sync + from xberg import ChunkingConfig, ExtractionConfig + + from lilbee.data.xberg_extract import extract_document if heading_context: max_chars, max_overlap = _char_budget() @@ -81,7 +83,7 @@ def chunk_text( chunking = build_chunking_config(use_semantic=use_semantic) config = ExtractionConfig(chunking=chunking) - result = extract_bytes_sync(text.encode("utf-8"), mime_type, config=config) - if result.chunks: - return [c.content for c in result.chunks] + doc = extract_document(text.encode("utf-8"), mime_type, config=config) + if doc.chunks: + return [c.content for c in doc.chunks] return [] diff --git a/src/lilbee/data/ingest/extract.py b/src/lilbee/data/ingest/extract.py index 1a952bfa7..77ee5cf7a 100644 --- a/src/lilbee/data/ingest/extract.py +++ b/src/lilbee/data/ingest/extract.py @@ -32,11 +32,7 @@ ) if TYPE_CHECKING: - from xberg import ExtractionConfig, OcrConfig - - # extract_* return the pyo3 result (attribute access), not the public - # ExtractionResult TypedDict (xberg-7ih). - from xberg._xberg import ExtractionResult + from xberg import ExtractedDocument, ExtractionConfig, OcrConfig log = logging.getLogger(__name__) @@ -188,25 +184,25 @@ async def chunk_and_embed_pages( def _capture_result_page_texts( - result: ExtractionResult, + doc: ExtractedDocument, source_name: str, content_type: str, page_texts_out: list[PageTextRecord] | None, ) -> None: """Append an extraction's page texts to the export accumulator. - Paginated documents yield one row per ``result.pages`` entry; others have no - page split, so the full ``result.content`` is recorded as page 0. + Paginated documents yield one row per ``doc.pages`` entry; others have no + page split, so the full ``doc.content`` is recorded as page 0. """ if page_texts_out is None: return - if result.pages: + if doc.pages: page_texts_out.extend( _page_text_record(source_name, page.page_number, page.content, content_type) - for page in result.pages + for page in doc.pages ) - elif result.content.strip(): - page_texts_out.append(_page_text_record(source_name, 0, result.content, content_type)) + elif doc.content.strip(): + page_texts_out.append(_page_text_record(source_name, 0, doc.content, content_type)) def _warn_empty_ocr(source_name: str, media: str) -> None: @@ -237,7 +233,7 @@ async def ingest_document( pipeline call compatibility. """ del quiet - from xberg import extract_file + from lilbee.data.xberg_extract import aextract_document page_seen = 0 @@ -251,26 +247,26 @@ def _tick() -> None: with ocr_request(on_page=_tick, timeout=_effective_ocr_timeout()) as token: config = extraction_config(content_type_to_mode(content_type), ocr_token=token) - # Async keeps the OCR page loop off this event loop's thread. - result = await extract_file(str(path), config=config) + # xberg's extract is async; awaiting it keeps the OCR page loop off this thread. + doc = await aextract_document(path.read_bytes(), filename=path.name, config=config) - if not result.chunks: + if not doc.chunks: if content_type in (PDF_CONTENT_TYPE, IMAGE_CONTENT_TYPE): _warn_empty_ocr(source_name, "scanned documents") return [] - _capture_result_page_texts(result, source_name, content_type, page_texts_out) + _capture_result_page_texts(doc, source_name, content_type, page_texts_out) # One EXTRACT event per file so subscribers (chat /add, /sync, CLI Rich # progress) show "extracted N pages" before the embed phase; result.pages is # the canonical page list, falling back to the chunk count for non-paginated docs. - page_count = len(result.pages or []) or len(result.chunks or []) + page_count = len(doc.pages or []) or len(doc.chunks or []) on_progress( EventType.EXTRACT, ExtractEvent(file=source_name, page=page_count, total_pages=page_count), ) - texts = [chunk.content for chunk in result.chunks] + texts = [chunk.content for chunk in doc.chunks] vectors = await asyncio.to_thread( get_services().embedder.embed_batch, texts, source=source_name, on_progress=on_progress ) @@ -287,7 +283,7 @@ def _tick() -> None: chunk_index=chunk.metadata.chunk_index, vector=vec, ) - for chunk, text, vec in zip(result.chunks, texts, vectors, strict=True) + for chunk, text, vec in zip(doc.chunks, texts, vectors, strict=True) ] diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py index 8eda1e056..8a786bc53 100644 --- a/src/lilbee/data/ingest/vision_ocr_backend.py +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -71,16 +71,16 @@ def ocr_request( ocr_requests.unregister(token) -def backend_options_for(token: str) -> str: - """Serialize a request token into the OcrConfig.backend_options string.""" - return json.dumps({_REQUEST_TOKEN_KEY: token}) +def backend_options_for(token: str) -> dict[str, str]: + """Carry a request token in OcrConfig.backend_options for process_image to read.""" + return {_REQUEST_TOKEN_KEY: token} class _OcrConfigView: """Typed reader over the xberg OcrConfig object passed to process_image. xberg hands the callback a native OcrConfig (alef typed trait callbacks), so - its fields are read directly as attributes. + its fields are read directly as attributes; ``backend_options`` is a dict. """ def __init__(self, config: OcrConfig) -> None: @@ -92,13 +92,7 @@ def vlm_prompt(self) -> str | None: @property def request_token(self) -> str | None: - raw = self._config.backend_options - if raw is None: - return None - try: - options = json.loads(raw) - except (ValueError, TypeError): - return None + options = self._config.backend_options token = options.get(_REQUEST_TOKEN_KEY) if isinstance(options, dict) else None return token if isinstance(token, str) else None @@ -144,9 +138,12 @@ def shutdown(self) -> None: ... def backend_type(self) -> str: return "custom" - def process_image(self, image_bytes: bytes, config: OcrConfig) -> dict[str, Any]: - # xberg passes the OcrConfig as a native object (alef typed trait - # callbacks); read the request token and prompt override off it. + def process_image(self, image_bytes: bytes, config: OcrConfig) -> ExtractedDocument: + # xberg passes the OcrConfig as a native object and expects a native + # ExtractedDocument back (alef typed trait callbacks); read the request + # token and prompt override off the config, return the OCR text as markdown. + from xberg import ExtractedDocument + view = _OcrConfigView(config) model = self._model_ref_fn() prompt = view.vlm_prompt or resolve_ocr_prompt(model) @@ -154,13 +151,4 @@ def process_image(self, image_bytes: bytes, config: OcrConfig) -> dict[str, Any] text = self._ocr_fn(image_bytes, model, prompt, timeout=ctx.timeout if ctx else 0.0) if ctx is not None and ctx.on_page is not None: ctx.on_page() - # xberg deserializes this dict into its OCR result struct; all of these - # keys are required (xberg-1mc). - return { - "content": text, - "mime_type": MARKDOWN_MIME, - "metadata": {}, - "tables": [], - "chunks": [], - "images": [], - } + return ExtractedDocument(content=text, mime_type=MARKDOWN_MIME) diff --git a/src/lilbee/data/xberg_extract.py b/src/lilbee/data/xberg_extract.py new file mode 100644 index 000000000..d8e49c8d5 --- /dev/null +++ b/src/lilbee/data/xberg_extract.py @@ -0,0 +1,75 @@ +"""Bridge to xberg's async-only ``extract`` for lilbee's call sites. + +xberg 1.x exposes a single ``extract(ExtractInput, config) -> ExtractionResult`` +coroutine whose ``results`` hold one ``ExtractedDocument`` per input. lilbee +extracts one in-memory document at a time, from both async code (await directly) +and synchronous code (driven to completion here). +""" + +from __future__ import annotations + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Coroutine + + from xberg import ExtractedDocument, ExtractionConfig, ExtractionResult + + +def _input(data: bytes, mime_type: str | None, filename: str | None): + from xberg import ExtractInput, ExtractInputKind + + return ExtractInput( + kind=ExtractInputKind.BYTES, bytes=data, mime_type=mime_type, filename=filename + ) + + +def _first(result: ExtractionResult) -> ExtractedDocument: + """Return the single extracted document, or raise on an extraction error.""" + if result.results: + return result.results[0] + if result.errors: + raise RuntimeError(str(result.errors[0])) + raise RuntimeError("xberg extraction returned no document") + + +async def aextract_document( + data: bytes, + mime_type: str | None = None, + *, + filename: str | None = None, + config: ExtractionConfig, +) -> ExtractedDocument: + """Extract one in-memory document. For callers already on the event loop.""" + from xberg import extract + + return _first(await extract(_input(data, mime_type, filename), config)) + + +def extract_document( + data: bytes, + mime_type: str | None = None, + *, + filename: str | None = None, + config: ExtractionConfig, +) -> ExtractedDocument: + """Extract one in-memory document from synchronous code. + + Drives xberg's coroutine to completion. When no event loop is running on this + thread (the common case: a plain sync caller or one of lilbee's offloaded + worker threads) ``asyncio.run`` is used directly; if a loop is already running + here, the coroutine is driven on a fresh worker thread so it never re-enters + that loop. + """ + return _run(aextract_document(data, mime_type, filename=filename, config=config)) + + +def _run(coro: Coroutine[None, None, ExtractedDocument]) -> ExtractedDocument: + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + with ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, coro).result() diff --git a/uv.lock b/uv.lock index 8a586a832..3eeb3921c 100644 --- a/uv.lock +++ b/uv.lock @@ -1681,7 +1681,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "typing-extensions", specifier = ">=4.5.0" }, { name = "uvicorn", specifier = ">=0.30" }, - { name = "xberg", path = "../../../../xberg-all-prs-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" }, + { name = "xberg", path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl" }, ] provides-extras = ["litellm", "graph", "crawler", "cuda12", "release"] @@ -4369,9 +4369,9 @@ wheels = [ [[package]] name = "xberg" version = "1.0.0rc1" -source = { path = "../../../../xberg-all-prs-wheel/xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl" } +source = { path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:da563ff860a17ef74ed66f25a6d45e97de533dc578883258ef995138f89a1635" }, + { filename = "xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4657dec8770a21cd4e4e37618583df969360350f668bb391e337ead3c0585a64" }, ] [[package]] From 696021bb5ed8b118a3c0a1d00e10b44f3062c3d4 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 29 Jun 2026 01:41:31 -0400 Subject: [PATCH 16/77] fix(deps): update xberg wheel hash to match rebuilt rc1 wheel --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 3eeb3921c..64c1c73f4 100644 --- a/uv.lock +++ b/uv.lock @@ -4371,7 +4371,7 @@ name = "xberg" version = "1.0.0rc1" source = { path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4657dec8770a21cd4e4e37618583df969360350f668bb391e337ead3c0585a64" }, + { filename = "xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e4358520a335bd879750a85761378c086cba7e9da4bf78669b5d5781493edfad" }, ] [[package]] From fea490cc6f62471ee048083cee916630efffcfd2 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 29 Jun 2026 02:16:14 -0400 Subject: [PATCH 17/77] =?UTF-8?q?fix(xberg):=20finish=20new-API=20migratio?= =?UTF-8?q?n=20=E2=80=94=20test=20mocks,=20OCR=20return=20type,=20obsolete?= =?UTF-8?q?=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete the lilbee port to xberg's async extract()/ExtractedDocument API now that the rebuilt wheel fixes the OCR SIGBUS (kreuzberg-6a2) and accepts dict backend_options (kreuzberg-9n0): - vision_ocr_backend: process_image returns a native ExtractedDocument; backend_options read as a dict; drop the unused json import - xberg_extract: typed _input/_run helpers - services: register_ocr_backend arg-type ignore (xberg's OcrBackend Protocol lists file/document methods an image backend never needs; duck-typed at runtime) - tests: repoint extract mocks to the xberg_extract bridge, assert ExtractedDocument + dict backend_options, create files for read_bytes; drop tests for upstream-removed pdf_ocr and elastic ingest-pool (#447) --- src/lilbee/app/services.py | 4 +- src/lilbee/data/ingest/vision_ocr_backend.py | 3 +- src/lilbee/data/xberg_extract.py | 4 +- tests/server/test_handlers.py | 27 +-- tests/test_chunker.py | 2 +- tests/test_fleet_provider.py | 190 ------------------- tests/test_formats.py | 12 +- tests/test_ingest.py | 96 +++++++--- tests/test_vision_ocr_backend.py | 16 +- 9 files changed, 101 insertions(+), 253 deletions(-) diff --git a/src/lilbee/app/services.py b/src/lilbee/app/services.py index cd87fd45e..02ac66994 100644 --- a/src/lilbee/app/services.py +++ b/src/lilbee/app/services.py @@ -216,8 +216,10 @@ def sync_vision_ocr_backend(provider: LLMProvider) -> None: # Re-register so the backend always binds to the current provider. if registered: unregister_ocr_backend(OcrBackendName.LILBEE_VISION) + # xberg's OcrBackend Protocol lists file/document methods an image-only + # backend never needs; registration is duck-typed and validated at runtime. register_ocr_backend( - VisionOcrBackend(ocr_fn=provider.vision_ocr, model_ref_fn=lambda: cfg.vision_model) + VisionOcrBackend(ocr_fn=provider.vision_ocr, model_ref_fn=lambda: cfg.vision_model) # type: ignore[arg-type] ) elif registered: unregister_ocr_backend(OcrBackendName.LILBEE_VISION) diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py index 8a786bc53..2ebbf430e 100644 --- a/src/lilbee/data/ingest/vision_ocr_backend.py +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import threading import uuid from contextlib import contextmanager @@ -16,7 +15,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator - from xberg import OcrConfig + from xberg import ExtractedDocument, OcrConfig # Token key inside OcrConfig.backend_options JSON. xberg does not propagate # contextvars into process_image (xberg-4w9), so per-request state travels as diff --git a/src/lilbee/data/xberg_extract.py b/src/lilbee/data/xberg_extract.py index d8e49c8d5..26d171d43 100644 --- a/src/lilbee/data/xberg_extract.py +++ b/src/lilbee/data/xberg_extract.py @@ -15,10 +15,10 @@ if TYPE_CHECKING: from collections.abc import Coroutine - from xberg import ExtractedDocument, ExtractionConfig, ExtractionResult + from xberg import ExtractedDocument, ExtractInput, ExtractionConfig, ExtractionResult -def _input(data: bytes, mime_type: str | None, filename: str | None): +def _input(data: bytes, mime_type: str | None, filename: str | None) -> ExtractInput: from xberg import ExtractInput, ExtractInputKind return ExtractInput( diff --git a/tests/server/test_handlers.py b/tests/server/test_handlers.py index 3257c5773..15209f61d 100644 --- a/tests/server/test_handlers.py +++ b/tests/server/test_handlers.py @@ -4,7 +4,6 @@ import contextlib from pathlib import Path from unittest import mock -from unittest.mock import Mock import pytest from litestar.testing import AsyncTestClient @@ -91,7 +90,11 @@ def _make_xberg_result(text: str = "Some extracted text. " * 20, num_chunks: int return result -@mock.patch("xberg.extract_file_sync", new_callable=Mock, return_value=_make_xberg_result()) +@mock.patch( + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), +) class TestAddEndpoint: async def test_add_single_file(self, mock_extract_file, isolated_env, tmp_path): """POST /api/add with a valid file streams SSE events and adds it.""" @@ -271,8 +274,8 @@ async def test_exactly_max_files_accepted(self, isolated_env, tmp_path): paths = [f"/fake/file_{i}.txt" for i in range(MAX_ADD_FILES)] with mock.patch( - "xberg.extract_file_sync", - new_callable=Mock, + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ): async with AsyncTestClient(create_app()) as client: @@ -537,8 +540,8 @@ async def test_partial_contention_partitions_paths(self, isolated_env, tmp_path) assert lock is not None try: with mock.patch( - "xberg.extract_file_sync", - new_callable=Mock, + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ): events = await self._collect(add_files_stream([str(held), str(free)])) @@ -567,8 +570,8 @@ async def test_concurrent_different_sources_run_in_parallel(self, isolated_env, async def _run(path: Path): text = "" with mock.patch( - "xberg.extract_file_sync", - new_callable=Mock, + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ): async for frame in add_files_stream([str(path)]): @@ -652,8 +655,8 @@ async def test_new_file_triggers_cleanup(self, isolated_env, tmp_path, mock_svc) store.get_sources.return_value = [] with mock.patch( - "xberg.extract_file_sync", - new_callable=Mock, + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ): await sync(quiet=True) @@ -676,8 +679,8 @@ async def test_retry_after_orphaned_chunks_cleans_up(self, isolated_env, tmp_pat store.get_sources.return_value = [] with mock.patch( - "xberg.extract_file_sync", - new_callable=Mock, + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ): await sync(quiet=True) diff --git a/tests/test_chunker.py b/tests/test_chunker.py index e54a95d9a..f2b43374f 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -591,5 +591,5 @@ def test_returns_empty_when_no_chunks(self): mock_result = MagicMock() mock_result.chunks = [] - with patch("xberg.extract_bytes_sync", return_value=mock_result): + with patch("lilbee.data.xberg_extract.extract_document", return_value=mock_result): assert chunk_text("some text") == [] diff --git a/tests/test_fleet_provider.py b/tests/test_fleet_provider.py index e4668f0b1..8b139e42b 100644 --- a/tests/test_fleet_provider.py +++ b/tests/test_fleet_provider.py @@ -672,196 +672,6 @@ def test_supports_tools_tolerates_unstattable_path(monkeypatch, _clear_tools_cac assert FleetProvider().supports_tools("org/repo/chat.gguf") is True -def test_release_ingest_pool_unloads_only_elastic_replicas() -> None: - from unittest import mock - - from lilbee.providers.fleet.provider import FleetProvider - - p = FleetProvider() - p._elastic_members = ["embed-1", "embed-2", "vision-1"] - unloaded: list[str] = [] - swap = mock.Mock() - swap.unload.side_effect = lambda mid: unloaded.append(mid) or True - p._swap = swap - - p.release_ingest_pool() - - assert unloaded == ["embed-1", "embed-2", "vision-1"] - swap.unload.assert_called() - - -def test_release_ingest_pool_noop_when_no_swap() -> None: - from lilbee.providers.fleet.provider import FleetProvider - - p = FleetProvider() - p._elastic_members = ["embed-1"] - p._swap = None - p.release_ingest_pool() # must not raise - - -def test_release_ingest_pool_logs_unconfirmed(caplog) -> None: - import logging - from unittest import mock - - from lilbee.providers.fleet.provider import FleetProvider - - p = FleetProvider() - p._elastic_members = ["embed-1"] - swap = mock.Mock() - swap.unload.return_value = False # unload did not confirm - p._swap = swap - - with caplog.at_level(logging.INFO, logger="lilbee.providers.fleet.provider"): - p.release_ingest_pool() - - assert any("did not confirm" in r.message for r in caplog.records) - - -def test_adopt_swap_records_elastic_members(monkeypatch) -> None: - from lilbee.providers.fleet.launch import InstanceLaunch - from lilbee.providers.fleet.provider import FleetProvider - - def _make_launch(role: WorkerRole, replica: int) -> InstanceLaunch: - return InstanceLaunch( - role=role, - argv=[], - env_overrides={}, - model="dummy.gguf", - replica=replica, - ) - - launches = [ - _make_launch(WorkerRole.CHAT, replica=0), - _make_launch(WorkerRole.EMBED, replica=0), - _make_launch(WorkerRole.EMBED, replica=1), - _make_launch(WorkerRole.RERANK, replica=0), - _make_launch(WorkerRole.VISION, replica=0), - _make_launch(WorkerRole.VISION, replica=1), - ] - swap = _install_engine(monkeypatch, launches=launches) - p = FleetProvider() - with p._lock: - p._adopt_swap(swap, launches) - - assert p._elastic_members == ["embed-1", "vision-1"] - - -def test_pdf_ocr_ocrs_each_page_over_vision_server(monkeypatch) -> None: - from lilbee.runtime.progress import EventType - from lilbee.vision import PageText - - client = _fake_client(0) - client.chat.side_effect = ["page one", "page two"] - p = _provider_with_clients({WorkerRole.VISION: [client]}) - monkeypatch.setattr(cfg, "vision_model", "") # empty model arg -> configured - monkeypatch.setattr(cfg, "vision_ocr_concurrency", 1) # sequential: side_effect by call order - monkeypatch.setattr("lilbee.vision.pdf_page_count", lambda _p: 2) - monkeypatch.setattr( - "lilbee.vision.rasterize_pdf", lambda _p: iter([(0, b"png0"), (1, b"png1")]) - ) - events: list[tuple] = [] - result = p.pdf_ocr( - Path("doc.pdf"), - backend="vision", # type: ignore[arg-type] - on_progress=lambda etype, evt: events.append((etype, evt.page, evt.total_pages)), - ) - assert result == [PageText(1, "page one"), PageText(2, "page two")] - assert events == [(EventType.EXTRACT, 1, 2), (EventType.EXTRACT, 2, 2)] - - -def test_pdf_ocr_runs_pages_concurrently_and_preserves_order(monkeypatch) -> None: - # OCR fans pages across the vision server's batching slots; results must still - # come back in page order, and more than one page must be in flight at once. - import threading - import time as _time - - monkeypatch.setattr(cfg, "vision_model", "") - monkeypatch.setattr(cfg, "vision_ocr_concurrency", 4) - # Auto replicas (vision_replicas left at its 0 default) resolves to one per - # GPU; pin the probe to a single GPU so the gate admits 1 x 4 = 4 in flight. - monkeypatch.setattr(prov_mod, "gpu_device_count", lambda: 1) - n = 8 - monkeypatch.setattr("lilbee.vision.pdf_page_count", lambda _p: n) - monkeypatch.setattr( - "lilbee.vision.rasterize_pdf", lambda _p: iter([(i, f"png{i}".encode()) for i in range(n)]) - ) - lock = threading.Lock() - inflight = {"now": 0, "max": 0} - - def _vision(_client, _messages, _timeout): - with lock: - inflight["now"] += 1 - inflight["max"] = max(inflight["max"], inflight["now"]) - _time.sleep(0.02) - with lock: - inflight["now"] -= 1 - return "ocr" - - monkeypatch.setattr(prov_mod, "_vision_call", _vision) - p = _provider_with_clients({WorkerRole.VISION: [_fake_client(0)]}) - result = p.pdf_ocr(Path("doc.pdf"), backend="vision") # type: ignore[arg-type] - assert [pt.page for pt in result] == list(range(1, n + 1)) # reassembled in order - assert inflight["max"] >= 2 # pages ran concurrently, not one at a time - - -def test_pdf_drain_budget_totals_pages_plus_load_grace(monkeypatch) -> None: - """Budget is one document-wide pool: pages*per_page + load grace, else uncapped.""" - monkeypatch.setattr(cfg, "vision_load_budget_s", 300.0) - assert prov_mod._pdf_drain_budget(2, 120.0) == 540.0 - assert prov_mod._pdf_drain_budget(5, None) is None - assert prov_mod._pdf_drain_budget(5, 0.0) is None - - -def test_pdf_ocr_spends_one_document_budget_across_pages(monkeypatch) -> None: - """Each page gets the remaining doc budget, not a fixed per-page cap.""" - from lilbee.vision import PageText - - p = _provider_with_clients({WorkerRole.VISION: [_fake_client(0)]}) - monkeypatch.setattr(cfg, "vision_model", "") - monkeypatch.setattr(cfg, "vision_load_budget_s", 300.0) - # Pin the device-count probe so _checkout() does not run a real subprocess - # and spend ~4s that would bleed into the budget timing assertion. - monkeypatch.setattr(prov_mod, "gpu_device_count", lambda: 1) - monkeypatch.setattr("lilbee.vision.pdf_page_count", lambda _p: 2) - monkeypatch.setattr( - "lilbee.vision.rasterize_pdf", lambda _p: iter([(0, b"png0"), (1, b"png1")]) - ) - seen: list[float | None] = [] - - def _capture(_client, _messages, timeout): - seen.append(timeout) - return "ocr" - - monkeypatch.setattr(prov_mod, "_vision_call", _capture) - result = p.pdf_ocr(Path("doc.pdf"), backend="vision", per_page_timeout_s=120.0) # type: ignore[arg-type] - assert result == [PageText(1, "ocr"), PageText(2, "ocr")] - # Budget is 2*120 + 300 = 540; pages run concurrently, so each draws nearly - # the full remaining budget (far above any 120 cap), in either capture order. - assert seen[0] == pytest.approx(540.0, abs=1.0) - assert seen[1] == pytest.approx(540.0, abs=1.0) - assert all(t is not None and t > 120.0 for t in seen) - - -def test_pdf_ocr_without_per_page_timeout_runs_uncapped(monkeypatch) -> None: - """No per-page cap means an uncapped (None) budget on every page.""" - p = _provider_with_clients({WorkerRole.VISION: [_fake_client(0)]}) - monkeypatch.setattr(cfg, "vision_model", "") - monkeypatch.setattr("lilbee.vision.pdf_page_count", lambda _p: 1) - monkeypatch.setattr("lilbee.vision.rasterize_pdf", lambda _p: iter([(0, b"png0")])) - seen: list[float | None] = [] - monkeypatch.setattr(prov_mod, "_vision_call", lambda *a: seen.append(a[2]) or "ocr") - p.pdf_ocr(Path("doc.pdf"), backend="vision", per_page_timeout_s=None) # type: ignore[arg-type] - assert seen == [None] - - -def test_pdf_ocr_without_server_raises() -> None: - from lilbee.providers.base import ProviderError - - p = _provider_with_clients({}) - with pytest.raises(ProviderError, match="No vision model server is running"): - p.pdf_ocr(Path("doc.pdf"), backend="vision") # type: ignore[arg-type] - - # --- llama-swap lifecycle ---------------------------------------------------- diff --git a/tests/test_formats.py b/tests/test_formats.py index f19ceec70..454688856 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -79,7 +79,7 @@ def _make_xberg_result(text="Extracted content. " * 10, num_chunks=1): @mock.patch( - "xberg.extract_file", + "lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ) @@ -93,7 +93,7 @@ async def test_docx_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "xberg.extract_file", + "lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ) @@ -107,7 +107,7 @@ async def test_xlsx_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "xberg.extract_file", + "lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ) @@ -126,7 +126,7 @@ async def test_pptx_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "xberg.extract_file", + "lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ) @@ -145,7 +145,7 @@ async def test_epub_discovered_and_ingested(self, mock_extract_file, isolated_en @mock.patch( - "xberg.extract_file", + "lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ) @@ -224,7 +224,7 @@ async def test_code_file_syncs(self, isolated_env, filename, fixture): @mock.patch( - "xberg.extract_file", + "lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ) diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 50e7a4b4d..833621d63 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -176,7 +176,11 @@ def _make_empty_result(): return result -@mock.patch("xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result()) +@mock.patch( + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), +) class TestSync: async def test_empty_documents_dir(self, mock_extract_file, isolated_env): from lilbee.data.ingest import SyncResult, sync @@ -681,7 +685,11 @@ async def _fail(path, name, ct, **kwargs): assert "qflaky.txt" not in result.updated -@mock.patch("xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result()) +@mock.patch( + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), +) class TestSyncCancellation: """Tests for cancel support and atomic per-file delete in sync.""" @@ -824,7 +832,9 @@ class TestIngestHelpers: """Cover edge cases in ingest_document and ingest_code_sync.""" @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_empty_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_empty_result(), ) async def testingest_document_empty_chunks(self, mock_extract_file, isolated_env): """Document that produces no chunks returns empty list.""" @@ -879,7 +889,7 @@ class _FakeResult: assert "# File: pkg/mod.py" in joined assert str(f) not in joined # absolute path must never leak into content - @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) async def testingest_document_pdf_with_pages(self, mock_kf, isolated_env): """PDF document returns records with page metadata.""" mock_kf.return_value = _make_xberg_result( @@ -899,7 +909,9 @@ async def testingest_document_pdf_with_pages(self, mock_kf, isolated_env): class TestCancellation: @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_cancelled_error_propagates(self, mock_extract_file, isolated_env): """CancelledError in _process_one is re-raised, not swallowed.""" @@ -920,7 +932,9 @@ async def _cancel(*args, **kwargs): await ingest_batch([entry], added, {}, {}, {}, quiet=True) @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_task_cancelled_error_does_not_orphan_siblings( self, mock_extract_file, isolated_env @@ -1316,7 +1330,7 @@ def _spy_plan(*args, **kwargs): monkeypatch.setattr(pipeline, "_plan_file_changes", _spy_plan) with mock.patch( - "xberg.extract_file", + "lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ): @@ -1333,7 +1347,7 @@ async def test_sync_backfills_stats_via_store(self, isolated_env, mock_svc): mock_svc.store.upsert_source("legacy.txt", file_hash(f), 1, source_type="document") with mock.patch( - "xberg.extract_file", + "lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock, return_value=_make_xberg_result(), ): @@ -2037,7 +2051,11 @@ def test_classify(self, filename, expected): assert classify_file(Path(filename)) == expected -@mock.patch("xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result()) +@mock.patch( + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), +) class TestSyncStructuredFormats: async def test_xml_file_ingested( self, @@ -2246,7 +2264,7 @@ async def test_frontmatter_only_produces_chunks(self, isolated_env): class TestPageTextAccumulator: """`page_texts_out` captures clean per-page text for the export dataset.""" - @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) async def test_pdf_pages_captured(self, mock_kf, isolated_env): mock_kf.return_value = _make_xberg_result(num_chunks=2, has_pages=True) from lilbee.data.ingest import ingest_document @@ -2258,7 +2276,7 @@ async def test_pdf_pages_captured(self, mock_kf, isolated_env): assert [p["page"] for p in pages] == [1, 2] assert all(p["content_type"] == "pdf" for p in pages) - @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) async def test_non_paginated_doc_captured_as_page_zero(self, mock_kf, isolated_env): mock_kf.return_value = _make_xberg_result(text="Plain body. " * 10, has_pages=False) from lilbee.data.ingest import ingest_document @@ -2283,7 +2301,7 @@ async def test_markdown_captured_as_page_zero(self, isolated_env): assert pages[0]["page"] == 0 assert "markdown body" in pages[0]["text"] - @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) async def test_ocr_pages_captured(self, mock_kf, isolated_env, mock_svc): # xberg OCRs scanned pages in-pass; the OCR'd text arrives in result.pages. cfg.enable_ocr = True @@ -2306,18 +2324,20 @@ async def test_empty_extraction_returns_empty(self, isolated_env): """Structured formats now go through xberg: empty result yields no chunks.""" from lilbee.data.ingest import ingest_document + (isolated_env / "e.xml").write_bytes(b"") # ingest_document reads the file bytes empty_result = mock.MagicMock(chunks=[]) mock_extract = mock.AsyncMock(return_value=empty_result) - with mock.patch("xberg.extract_file", mock_extract): + with mock.patch("lilbee.data.xberg_extract.aextract_document", mock_extract): result = await ingest_document(isolated_env / "e.xml", "e.xml", "xml") assert result == [] async def test_no_chunks_returns_empty(self, isolated_env): from lilbee.data.ingest import ingest_document + (isolated_env / "s.xml").write_bytes(b"") # ingest_document reads the file bytes no_chunks_result = mock.MagicMock(chunks=[]) mock_extract = mock.AsyncMock(return_value=no_chunks_result) - with mock.patch("xberg.extract_file", mock_extract): + with mock.patch("lilbee.data.xberg_extract.aextract_document", mock_extract): result = await ingest_document(isolated_env / "s.xml", "s.xml", "xml") assert result == [] @@ -2342,7 +2362,9 @@ def _mock_concepts_available(self): yield @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_concept_extraction_called_during_ingest( self, mock_extract_file, isolated_env, mock_svc @@ -2359,7 +2381,9 @@ async def test_concept_extraction_called_during_ingest( mock_svc.concepts.extract_concepts_batch.assert_called() @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_concept_disabled_skips_extraction( self, mock_extract_file, isolated_env, mock_svc @@ -2374,7 +2398,9 @@ async def test_concept_disabled_skips_extraction( mock_svc.concepts.extract_concepts_batch.assert_not_called() @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_concept_failure_does_not_break_ingest( self, mock_extract_file, isolated_env, mock_svc @@ -2391,7 +2417,9 @@ async def test_concept_failure_does_not_break_ingest( assert "concept_test2.txt" in result.added @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_concept_write_failure_does_not_fail_files( self, mock_extract_file, isolated_env, mock_svc @@ -2413,7 +2441,9 @@ async def test_concept_write_failure_does_not_fail_files( assert result.failed == [] @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_cluster_rebuild_called_after_sync( self, mock_extract_file, isolated_env, mock_svc @@ -2430,7 +2460,9 @@ async def test_cluster_rebuild_called_after_sync( mock_svc.concepts.rebuild_clusters.assert_called() @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_cluster_rebuild_failure_does_not_break_sync( self, mock_extract_file, isolated_env, mock_svc @@ -2448,7 +2480,9 @@ async def test_cluster_rebuild_failure_does_not_break_sync( assert "rebuild_test.txt" in result.added @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_graph_none_skips_indexing(self, mock_extract_file, isolated_env, mock_svc): """When get_graph() returns None, concept indexing is skipped gracefully.""" @@ -2462,7 +2496,9 @@ async def test_graph_none_skips_indexing(self, mock_extract_file, isolated_env, assert "graph_none_test.txt" in result.added @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_concepts_unavailable_skips_rebuild( self, mock_extract_file, isolated_env, mock_svc @@ -2479,7 +2515,9 @@ async def test_concepts_unavailable_skips_rebuild( mock_svc.concepts.rebuild_clusters.assert_not_called() @mock.patch( - "xberg.extract_file", new_callable=mock.AsyncMock, return_value=_make_xberg_result() + "lilbee.data.xberg_extract.aextract_document", + new_callable=mock.AsyncMock, + return_value=_make_xberg_result(), ) async def test_concepts_unavailable_skips_indexing( self, mock_extract_file, isolated_env, mock_svc @@ -2565,7 +2603,7 @@ def test_vision_backend_with_token_when_model_set(self, isolated_env): cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" config = extraction_config(ExtractMode.PAGINATED, ocr_token="tok-123") assert config["ocr"].backend == "lilbee-vision" - assert "tok-123" in config["ocr"].backend_options + assert config["ocr"].backend_options["req"] == "tok-123" class TestChunkAndEmbedPagesEmpty: @@ -2583,7 +2621,7 @@ async def test_whitespace_pages_yield_no_chunks(self, mock_svc): class TestIngestDocumentOcrPath: - @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) async def test_empty_pdf_warns_no_usable_text(self, mock_kf, isolated_env, mock_svc, caplog): mock_kf.return_value = mock.MagicMock(chunks=[]) from lilbee.data.ingest import ingest_document @@ -2594,18 +2632,16 @@ async def test_empty_pdf_warns_no_usable_text(self, mock_kf, isolated_env, mock_ assert result == [] assert "no usable text" in caplog.text - @mock.patch("xberg.extract_file", new_callable=mock.AsyncMock) + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) async def test_streams_per_page_progress_ticks(self, mock_kf, isolated_env, mock_svc): - import json - cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" result_obj = _make_xberg_result(num_chunks=1, has_pages=True) - async def fake_extract(path, *, config): + async def fake_extract(data, *, filename=None, config): # Simulate xberg calling the registered backend once per scanned page. from lilbee.data.ingest.vision_ocr_backend import ocr_requests - token = json.loads(config["ocr"].backend_options)["req"] + token = config["ocr"].backend_options["req"] ctx = ocr_requests.get(token) ctx.on_page() ctx.on_page() diff --git a/tests/test_vision_ocr_backend.py b/tests/test_vision_ocr_backend.py index 351098010..adbe2b722 100644 --- a/tests/test_vision_ocr_backend.py +++ b/tests/test_vision_ocr_backend.py @@ -71,17 +71,15 @@ def test_initialize_shutdown_noop(self): class TestProcessImage: - def test_returns_full_required_schema(self): + def test_returns_extracted_document(self): + """xberg expects a native ExtractedDocument back, with the OCR text as markdown.""" + from xberg import ExtractedDocument + be, _ = _backend() out = be.process_image(b"PNG", _cfg()) - assert out == { - "content": "# extracted", - "mime_type": MARKDOWN_MIME, - "metadata": {}, - "tables": [], - "chunks": [], - "images": [], - } + assert isinstance(out, ExtractedDocument) + assert out.content == "# extracted" + assert out.mime_type == MARKDOWN_MIME def test_passes_model_and_resolved_prompt(self): be, calls = _backend(model="vendor/glm-ocr") From 68cfcf7773e3a9843e28dcf17b3a7cea651d92d9 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 29 Jun 2026 02:22:44 -0400 Subject: [PATCH 18/77] test(xberg): cover the async-extract bridge error + offload branches --- tests/test_xberg_extract.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/test_xberg_extract.py diff --git a/tests/test_xberg_extract.py b/tests/test_xberg_extract.py new file mode 100644 index 000000000..28e7f4ae7 --- /dev/null +++ b/tests/test_xberg_extract.py @@ -0,0 +1,44 @@ +"""Tests for the xberg async-extract bridge (lilbee.data.xberg_extract).""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from lilbee.data import xberg_extract + + +class _FakeResult: + def __init__(self, results, errors=()): + self.results = list(results) + self.errors = list(errors) + + +def test_first_returns_single_document(): + doc = object() + assert xberg_extract._first(_FakeResult([doc])) is doc + + +def test_first_raises_on_extraction_error(): + with pytest.raises(RuntimeError, match="boom"): + xberg_extract._first(_FakeResult([], errors=["boom"])) + + +def test_first_raises_when_no_document(): + with pytest.raises(RuntimeError, match="no document"): + xberg_extract._first(_FakeResult([])) + + +@pytest.mark.asyncio +async def test_extract_document_offloads_when_a_loop_is_running(): + """Called from a thread with a live event loop, the sync bridge runs the + coroutine on a worker thread instead of re-entering the running loop.""" + doc = object() + + async def fake_extract(_input, _config): + return _FakeResult([doc]) + + with mock.patch("xberg.extract", fake_extract): + out = xberg_extract.extract_document(b"data", "text/plain", config=mock.MagicMock()) + assert out is doc From 6376e610a981de2fda0e466c75ccaedd5e43d3c3 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:38:23 -0400 Subject: [PATCH 19/77] build(xberg): integrate 1.0.0rc2 cp310-abi3 wheel Bump the local-dev xberg pin from the rc1 cp312-only wheel to the rc2 cp310-abi3 wheel. abi3 resolves across Python 3.11/3.12/3.13, so the integration no longer pins a single interpreter. Validated against rc2: all xberg imports resolve, image OCR and PDF-OCR-with-layout run end-to-end (the OCR future Box::pin fix is present in rc2, no SIGBUS), lint/typecheck/format clean. An A/B full-suite run (rc1-abi3 vs rc2-abi3) produced identical results, so rc2 is a behavior-for-behavior drop-in. --- pyproject.toml | 4 ++-- uv.lock | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 730d84745..b2adaf22a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,9 +92,9 @@ packages = ["src/lilbee"] # the index instead -- this source override is workspace metadata, not published deps. [tool.uv.sources] lilbee-engine = { path = "packaging/engine-wheel" } -# TODO: xberg 1.0.0rc1 is a pre-publication trial wheel; resolve it from the local +# TODO: xberg 1.0.0rc2 is a pre-publication trial wheel; resolve it from the local # build until it lands on PyPI, then drop this source and lock from the index. -xberg = { path = "/Users/tobias/projects/kreuzberg-wt-pywheel/dist/xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl" } +xberg = { path = "/Users/tobias/projects/kreuzberg-wt-pywheel/dist/xberg-1.0.0rc2-cp310-abi3-macosx_11_0_arm64.whl" } [dependency-groups] dev = [ diff --git a/uv.lock b/uv.lock index 64c1c73f4..f9d7e372d 100644 --- a/uv.lock +++ b/uv.lock @@ -1681,7 +1681,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "typing-extensions", specifier = ">=4.5.0" }, { name = "uvicorn", specifier = ">=0.30" }, - { name = "xberg", path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl" }, + { name = "xberg", path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc2-cp310-abi3-macosx_11_0_arm64.whl" }, ] provides-extras = ["litellm", "graph", "crawler", "cuda12", "release"] @@ -4368,10 +4368,10 @@ wheels = [ [[package]] name = "xberg" -version = "1.0.0rc1" -source = { path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl" } +version = "1.0.0rc2" +source = { path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc2-cp310-abi3-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e4358520a335bd879750a85761378c086cba7e9da4bf78669b5d5781493edfad" }, + { filename = "xberg-1.0.0rc2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:50577b30ceec53e4fcda990387a742066bb67722928590b661c70a1d221ad7fd" }, ] [[package]] From 0e6c959228409fcb040fae36138eaf30b4bf7ca5 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Wed, 1 Jul 2026 19:11:37 -0400 Subject: [PATCH 20/77] fix(config): honor LILBEE_SKIP_TOML_CONFIG on the overlay path; make tests hermetic The developer's real ~/Library/Application Support/lilbee/config.toml was bleeding into the test suite two ways: 1. Import-time load. The cfg singleton is built at import, before conftest's autouse fixture set LILBEE_SKIP_TOML_CONFIG, so values like vision_model and memory_enabled came from the dev's machine. Tests asserting defaults (e.g. 'no vision model -> tesseract OCR backend') failed. Fix: set the skip flag at conftest module top, before cfg is imported, matching CI (which has no config.toml). 2. CLI/MCP overlay. overlay_persisted_settings() re-read config.toml directly and ignored LILBEE_SKIP_TOML_CONFIG, unlike the pydantic-settings source, so a CLI invocation re-applied disk values mid-command (memory_enabled flipped False->True during 'memory add'). This is also a real product inconsistency: the documented escape hatch worked on one config-read path but not the other. Fix: overlay_persisted_settings() returns early when the flag is set, giving parity across import-time load, CLI callback, and MCP server. Tests that specifically exercise the overlay-applies path (test_cli TestApplyOverrides, test_mcp TestInit, test_settings TestOverlayPersistedSettings) opt back in via a new overlay_reads_config_toml fixture, and a regression test covers the skip-flag no-op. Also reformats a pre-existing line-wrap drift in ingest/pipeline.py that was failing format-check. Full suite 100% coverage. bb-e7d. --- src/lilbee/core/settings.py | 6 ++++++ src/lilbee/data/ingest/pipeline.py | 4 +--- tests/conftest.py | 20 ++++++++++++++++++++ tests/test_cli.py | 28 ++++++++++++++++++++-------- tests/test_mcp.py | 2 +- tests/test_settings.py | 22 +++++++++++++++++++++- 6 files changed, 69 insertions(+), 13 deletions(-) diff --git a/src/lilbee/core/settings.py b/src/lilbee/core/settings.py index 0ccf8693d..7faec2b61 100644 --- a/src/lilbee/core/settings.py +++ b/src/lilbee/core/settings.py @@ -93,7 +93,13 @@ def overlay_persisted_settings(root: Path) -> None: An explicit ``LILBEE_`` env var wins over config.toml (the documented precedence): cfg already holds the env-loaded value, so a key whose env var is set is left untouched rather than overwritten by the persisted file. + + ``LILBEE_SKIP_TOML_CONFIG=1`` disables this overlay entirely, matching the + pydantic-settings source in ``config/model.py`` so the escape hatch is honored + on every config-read path (import-time load, CLI callback, MCP server). """ + if os.environ.get("LILBEE_SKIP_TOML_CONFIG") == "1": + return log = logging.getLogger(__name__) try: persisted = load(root) diff --git a/src/lilbee/data/ingest/pipeline.py b/src/lilbee/data/ingest/pipeline.py index e8590beee..a607cc15d 100644 --- a/src/lilbee/data/ingest/pipeline.py +++ b/src/lilbee/data/ingest/pipeline.py @@ -390,9 +390,7 @@ async def sync( disk_files = discover_files() sources = _store.get_sources() existing_sources = {s["filename"]: s for s in sources} - skip_markers = _load_pruned_skip_markers( - disk_files, clear_first=force_rebuild or retry_skipped - ) + skip_markers = _load_pruned_skip_markers(disk_files, clear_first=force_rebuild or retry_skipped) removed: list[str] = [] failed: dict[str, None] = {} diff --git a/tests/conftest.py b/tests/conftest.py index f86b646cb..525be90a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,14 @@ # conftest runs, the sentinel half of the gate is satisfied here. os.environ.setdefault("LILBEE_SKIP_MODEL_TASK_VALIDATION", "1") +# Build the cfg singleton hermetically: the first ``import lilbee.core.config`` +# below constructs ``cfg`` from env + defaults, so skip the developer's real +# platform config.toml here (before that import) rather than only in an autouse +# fixture that runs after cfg is already loaded. Without this, values like +# ``vision_model`` / ``memory_enabled`` leak from the dev's machine into tests +# that assert defaults. Matches CI, which has no config.toml. (bb-e7d) +os.environ.setdefault("LILBEE_SKIP_TOML_CONFIG", "1") + from lilbee.catalog import CatalogModel from lilbee.catalog.refs import format_native_gguf_ref from lilbee.core.config import cfg @@ -186,6 +194,18 @@ def _ignore_user_global_config(monkeypatch): monkeypatch.setenv("LILBEE_SKIP_TOML_CONFIG", "1") +@pytest.fixture +def overlay_reads_config_toml(monkeypatch): + """Opt a test back into the config.toml overlay path. + + The suite runs with ``LILBEE_SKIP_TOML_CONFIG=1`` for hermeticity, and + ``overlay_persisted_settings`` honors that flag. Tests that specifically + exercise the overlay-applies behavior (writing a config.toml to a controlled + root and asserting it lands on cfg) must clear the flag so overlay runs. + """ + monkeypatch.delenv("LILBEE_SKIP_TOML_CONFIG", raising=False) + + @pytest.fixture(autouse=True) def _drain_textual_threads(): """Safety net: join non-daemon threads that outlive the test. diff --git a/tests/test_cli.py b/tests/test_cli.py index c5dd6b082..dd15f1fce 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -664,7 +664,7 @@ def test_generation_option_none_is_noop(self): assert cfg.temperature == 0.7 cfg.temperature = None - def test_data_dir_overlays_per_root_config_toml(self, tmp_path): + def test_data_dir_overlays_per_root_config_toml(self, tmp_path, overlay_reads_config_toml): """A per-vault config.toml in the data-dir must be re-read when --data-dir lands. Regression: cfg's scalar fields (chat_model, embedding_model, ...) were @@ -688,7 +688,9 @@ def test_data_dir_overlays_per_root_config_toml(self, tmp_path): assert cfg.chat_model == "ollama/qwen3:4b" assert cfg.embedding_model == "ollama/nomic-embed-text:v1.5" - def test_data_dir_without_config_toml_leaves_cfg_unchanged(self, tmp_path): + def test_data_dir_without_config_toml_leaves_cfg_unchanged( + self, tmp_path, overlay_reads_config_toml + ): """An empty / missing config.toml must not stomp on existing cfg values.""" from lilbee.cli import apply_overrides @@ -701,7 +703,9 @@ def test_data_dir_without_config_toml_leaves_cfg_unchanged(self, tmp_path): assert cfg.chat_model == "ollama/kept-from-import:latest" assert cfg.embedding_model == "ollama/kept-embed:latest" - def test_data_dir_overlay_covers_writable_scalar_fields(self, tmp_path): + def test_data_dir_overlay_covers_writable_scalar_fields( + self, tmp_path, overlay_reads_config_toml + ): """Writable scalar fields (e.g. temperature, top_k) overlay too, not just models.""" from lilbee.cli import apply_overrides @@ -715,7 +719,9 @@ def test_data_dir_overlay_covers_writable_scalar_fields(self, tmp_path): assert cfg.temperature == 0.2 assert cfg.top_k == 20 - def test_use_global_overlays_global_config_toml(self, tmp_path, monkeypatch): + def test_use_global_overlays_global_config_toml( + self, tmp_path, monkeypatch, overlay_reads_config_toml + ): """--global must also re-read the global root's config.toml.""" from lilbee.cli import apply_overrides @@ -731,7 +737,9 @@ def test_use_global_overlays_global_config_toml(self, tmp_path, monkeypatch): assert cfg.data_root == fake_global assert cfg.chat_model == "ollama/from-global:latest" - def test_lilbee_data_env_overlays_config_toml(self, tmp_path, monkeypatch): + def test_lilbee_data_env_overlays_config_toml( + self, tmp_path, monkeypatch, overlay_reads_config_toml + ): """The LILBEE_DATA env-var path must also overlay its config.toml.""" from lilbee.cli import apply_overrides @@ -775,7 +783,7 @@ def test_apply_data_root_exports_lilbee_data_env_global(self, tmp_path, monkeypa apply_overrides(use_global=True) assert os.environ.get("LILBEE_DATA") == str(fake_global) - def test_data_dir_overlay_skips_unknown_keys(self, tmp_path): + def test_data_dir_overlay_skips_unknown_keys(self, tmp_path, overlay_reads_config_toml): """Stale or unrecognised keys in config.toml don't blow up startup.""" from lilbee.cli import apply_overrides @@ -789,7 +797,9 @@ def test_data_dir_overlay_skips_unknown_keys(self, tmp_path): assert cfg.chat_model == "ollama/from-vault:latest" assert not hasattr(cfg, "totally_unknown_key") - def test_data_dir_overlay_logs_and_skips_invalid_value(self, tmp_path, caplog): + def test_data_dir_overlay_logs_and_skips_invalid_value( + self, tmp_path, caplog, overlay_reads_config_toml + ): """A malformed persisted value is logged and skipped, not raised.""" from lilbee.cli import apply_overrides @@ -803,7 +813,9 @@ def test_data_dir_overlay_logs_and_skips_invalid_value(self, tmp_path, caplog): assert cfg.top_k == 7 assert any("top_k" in rec.message for rec in caplog.records) - def test_data_dir_overlay_handles_unreadable_config_toml(self, tmp_path, monkeypatch, caplog): + def test_data_dir_overlay_handles_unreadable_config_toml( + self, tmp_path, monkeypatch, caplog, overlay_reads_config_toml + ): """A read failure on config.toml is logged and treated as 'no overlay'.""" from lilbee.cli import apply_overrides from lilbee.core import settings as settings_mod diff --git a/tests/test_mcp.py b/tests/test_mcp.py index b1c5a8987..afacddd85 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -485,7 +485,7 @@ def test_init_exports_lilbee_data_env(self, tmp_path, monkeypatch): init(str(target)) assert os.environ.get("LILBEE_DATA") == str(target) - def test_init_overlays_per_root_config_toml(self, tmp_path): + def test_init_overlays_per_root_config_toml(self, tmp_path, overlay_reads_config_toml): """init() must re-read the project base's config.toml, the same fix as the CLI's --data-dir entry point. Without this, switching the MCP session to a project that has its own model preferences silently diff --git a/tests/test_settings.py b/tests/test_settings.py index 9a62d0adf..8fd0935e6 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -356,11 +356,12 @@ def test_browser_memory_lever_defaults(self): class TestOverlayPersistedSettings: - def test_empty_string_value_is_skipped(self, tmp_path): + def test_empty_string_value_is_skipped(self, tmp_path, monkeypatch): """Legacy persisted empty strings (None written as "") skip overlay instead of corrupting the in-memory config or spamming warnings.""" from lilbee.core.config import cfg + monkeypatch.delenv("LILBEE_SKIP_TOML_CONFIG", raising=False) original = cfg.chat_model try: (tmp_path / "config.toml").write_text('chat_model = ""\n') @@ -375,6 +376,7 @@ def test_env_var_wins_over_config_toml(self, tmp_path, monkeypatch): original = cfg.vision_replicas try: + monkeypatch.delenv("LILBEE_SKIP_TOML_CONFIG", raising=False) cfg.vision_replicas = 4 # value as loaded from LILBEE_VISION_REPLICAS monkeypatch.setenv("LILBEE_VISION_REPLICAS", "4") (tmp_path / "config.toml").write_text("vision_replicas = 2\n") @@ -389,6 +391,7 @@ def test_empty_env_var_does_not_suppress_config_toml(self, tmp_path, monkeypatch original = cfg.vision_replicas try: + monkeypatch.delenv("LILBEE_SKIP_TOML_CONFIG", raising=False) monkeypatch.setenv("LILBEE_VISION_REPLICAS", "") cfg.vision_replicas = 1 (tmp_path / "config.toml").write_text("vision_replicas = 3\n") @@ -403,6 +406,7 @@ def test_config_toml_applies_when_env_absent(self, tmp_path, monkeypatch): original = cfg.vision_replicas try: + monkeypatch.delenv("LILBEE_SKIP_TOML_CONFIG", raising=False) monkeypatch.delenv("LILBEE_VISION_REPLICAS", raising=False) cfg.vision_replicas = 1 (tmp_path / "config.toml").write_text("vision_replicas = 3\n") @@ -411,6 +415,22 @@ def test_config_toml_applies_when_env_absent(self, tmp_path, monkeypatch): finally: cfg.vision_replicas = original + def test_skip_toml_config_makes_overlay_noop(self, tmp_path, monkeypatch): + """LILBEE_SKIP_TOML_CONFIG=1 disables the overlay path too, so the escape + hatch is honored consistently with the pydantic-settings source (the CLI + and MCP overlay must not re-read config.toml behind the skip flag).""" + from lilbee.core.config import cfg + + original = cfg.vision_replicas + try: + monkeypatch.setenv("LILBEE_SKIP_TOML_CONFIG", "1") + cfg.vision_replicas = 1 + (tmp_path / "config.toml").write_text("vision_replicas = 3\n") + settings.overlay_persisted_settings(tmp_path) + assert cfg.vision_replicas == 1 # config.toml ignored while skipping + finally: + cfg.vision_replicas = original + class TestAutoSyncConfig: def test_auto_sync_defaults_true(self): From 045c6b4fe96180294acdf3c5ea6ea47412e46395 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:27:01 -0400 Subject: [PATCH 21/77] test: reset xberg_extract module globals between tests to kill a cross-test mock leak chunk_text() flaked nondeterministically under the parallel suite, returning mock content ('Some extracted text.') instead of real output -- e.g. test_content_before_first_heading asserting 'Preamble' in the result, or test_long_text_produces_multiple_chunks getting a single chunk. Root cause: extract_document() (sync) resolves aextract_document via module-global lookup, so a mock.patch of lilbee.data.xberg_extract.aextract_document intercepts the sync path too, and chunk_text() uses the sync path. The ingest/handler suites patch that global at 50+ sites; under xdist a patch occasionally stays active into an unrelated test on the same worker (confirmed by a captured traceback; serial and small-xdist runs never leak). An autouse fixture now resets both extraction globals to their pristine functions before every test, so an escaped patch can't propagate. Verified: three consecutive full-suite runs clean after the change. bb-ql1. --- tests/conftest.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 525be90a1..7d668d85b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,10 +30,15 @@ from lilbee.catalog import CatalogModel from lilbee.catalog.refs import format_native_gguf_ref from lilbee.core.config import cfg +from lilbee.data import xberg_extract as _xberg_extract from lilbee.data.ingest import file_hash from lilbee.data.store import CitationRecord from lilbee.modelhub.registry import ModelManifest, ModelRegistry +# Pristine extraction entry points, captured before any test can patch them. +_PRISTINE_EXTRACT_DOCUMENT = _xberg_extract.extract_document +_PRISTINE_AEXTRACT_DOCUMENT = _xberg_extract.aextract_document + FIXTURES_DIR = Path(__file__).parent / "fixtures" @@ -194,6 +199,23 @@ def _ignore_user_global_config(monkeypatch): monkeypatch.setenv("LILBEE_SKIP_TOML_CONFIG", "1") +@pytest.fixture(autouse=True) +def _reset_xberg_extract_globals(): + """Start every test with the real extraction functions. + + ``extract_document`` (sync) resolves ``aextract_document`` via module-global + lookup, so a ``mock.patch`` of ``lilbee.data.xberg_extract.aextract_document`` + intercepts both the async and the sync path -- and the sync path is what + ``chunk_text`` uses. The ingest/handler suites patch that global heavily; under + the parallel run a patch occasionally stays active into an unrelated test, and + ``chunk_text`` then silently returns mock content ('Some extracted text.'), + a nondeterministic cross-test failure. Resetting the globals before each test + makes that leak impossible regardless of how a patch escaped. (bb-ql1) + """ + _xberg_extract.extract_document = _PRISTINE_EXTRACT_DOCUMENT + _xberg_extract.aextract_document = _PRISTINE_AEXTRACT_DOCUMENT + + @pytest.fixture def overlay_reads_config_toml(monkeypatch): """Opt a test back into the config.toml overlay path. From 5b6331754ab4c2ae210bcc74e0e19acc99d4307d Mon Sep 17 00:00:00 2001 From: Tobias <5562156+tobocop2@users.noreply.github.com> Date: Fri, 3 Jul 2026 04:03:53 -0400 Subject: [PATCH 22/77] Hoist unnecessary lazy cfg imports to module scope (#475) Six fleet/provider modules imported the config singleton lazily inside functions (planning.py alone had 13 copies; provider.py had 6 while already importing cfg at module level). None of these are load-bearing: cfg is built once and only mutated in place (never rebound), and none of these modules are pulled into the Config() construction chain, so a module-level import neither goes stale nor cycles. Consolidated each to a single module-level import, matching how registry.py and the TUI screens already import cfg. Also dropped a redundant function-local KvCacheType import in planning._cache_type_flag that duplicated the module-level one. Left the genuinely-needed lazy cfg imports in place: hf_client.py (documented config -> model_ref validator -> catalog -> hf_client cycle), gpu_env.py (a deliberately binding-free module), and services.py (its documented policy of deferring all imports to keep CLI startup fast). --- src/lilbee/providers/fleet/binary.py | 3 +-- src/lilbee/providers/fleet/gpu_select.py | 2 +- src/lilbee/providers/fleet/planning.py | 16 +--------------- src/lilbee/providers/fleet/provider.py | 7 ------- src/lilbee/providers/fleet/replicas.py | 2 +- src/lilbee/providers/sdk_backend.py | 2 +- 6 files changed, 5 insertions(+), 27 deletions(-) diff --git a/src/lilbee/providers/fleet/binary.py b/src/lilbee/providers/fleet/binary.py index 7ce3a786a..27c643e72 100644 --- a/src/lilbee/providers/fleet/binary.py +++ b/src/lilbee/providers/fleet/binary.py @@ -6,6 +6,7 @@ from enum import StrEnum from pathlib import Path +from lilbee.core.config import cfg from lilbee.providers.base import ProviderError _INSTALL_HINT = ( @@ -49,8 +50,6 @@ def resolve_engine_tool(tool: EngineTool) -> Path: wheel, then ``PATH``. """ if tool is EngineTool.LLAMA_SERVER: - from lilbee.core.config import cfg - if cfg.llama_server_path: configured = Path(cfg.llama_server_path) if not configured.is_file(): diff --git a/src/lilbee/providers/fleet/gpu_select.py b/src/lilbee/providers/fleet/gpu_select.py index a7c88314d..7599f684f 100644 --- a/src/lilbee/providers/fleet/gpu_select.py +++ b/src/lilbee/providers/fleet/gpu_select.py @@ -20,6 +20,7 @@ from dataclasses import dataclass from enum import IntEnum, StrEnum +from lilbee.core.config import cfg from lilbee.providers.fleet.vulkan_icd_discovery import ( iter_vulkan_manifest_paths, ) @@ -551,7 +552,6 @@ def disable_conflicting_vulkan_icds() -> str | None: from disk (registry on Windows, XDG on Linux); enumerating via ``vkCreateInstance`` would pre-load every vendor's ICD before the disable lands. """ - from lilbee.core.config import cfg if not _platform_supports_icd_pin(): return None diff --git a/src/lilbee/providers/fleet/planning.py b/src/lilbee/providers/fleet/planning.py index 9a57d8eb7..8e878c321 100644 --- a/src/lilbee/providers/fleet/planning.py +++ b/src/lilbee/providers/fleet/planning.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import TYPE_CHECKING +from lilbee.core.config import cfg from lilbee.core.config.enums import KvCacheType from lilbee.providers import model_cache from lilbee.providers.fleet.adapters import ( @@ -189,7 +190,6 @@ def _resolve_vision_slots( ) -> int: """Largest OCR batching slot count (<= ``cfg.vision_ocr_concurrency``) that fits the memory budget; 1 when the ceiling is 1 or nothing larger fits.""" - from lilbee.core.config import cfg ceiling = max(1, cfg.vision_ocr_concurrency) if ceiling == 1: @@ -228,7 +228,6 @@ def _slot_budget(vram_fraction: float, unified_budget: int | None) -> int: """Memory budget for slot sizing: *vram_fraction* of usable VRAM, capped by ``unified_budget`` (free system RAM) when there is no discrete GPU so the count steps down to fit free memory instead of overcommitting.""" - from lilbee.core.config import cfg budget = int(model_cache.get_available_memory(cfg.gpu_memory_fraction) * vram_fraction) if unified_budget is not None: @@ -271,7 +270,6 @@ def _role_ctx(role: WorkerRole, model_path: Path, meta: dict[str, str] | None) - falls back to the single-GPU dynamic chat-ctx picker. A tensor-split chat is sized against its per-device headroom instead (see :func:`fit_split_ctx`). """ - from lilbee.core.config import cfg from lilbee.providers.engine_params import ( resolve_chat_ctx, resolve_embed_ctx, @@ -294,7 +292,6 @@ def _role_ctx(role: WorkerRole, model_path: Path, meta: dict[str, str] | None) - def _rerank_mode_for(meta: dict[str, str] | None) -> RerankMode: """Resolve the RERANK serving mode from cfg + the reranker GGUF arch.""" - from lilbee.core.config import cfg arch = meta.get("architecture") if meta else None return resolve_rerank_mode(cfg.reranker_type, arch) @@ -334,7 +331,6 @@ def _role_gpu_layers(role: WorkerRole) -> int: def _flash_enabled() -> bool: """Flash attention is on unless ``cfg.flash_attention`` is explicitly ``False``.""" - from lilbee.core.config import cfg return cfg.flash_attention is not False @@ -351,7 +347,6 @@ def _role_flash(role: WorkerRole) -> bool: def _role_kv_cache_type(role: WorkerRole) -> KvCacheType: """Chat honors ``cfg.kv_cache_type``; embed/rerank/vision run f16 KV.""" - from lilbee.core.config import cfg return cfg.kv_cache_type if role is WorkerRole.CHAT else KvCacheType.F16 @@ -363,9 +358,6 @@ def _replica_count(role: WorkerRole, device_count: int) -> int: def _cache_type_flag() -> str | None: """KV cache type for chat, or ``None`` to leave llama-server's f16 default.""" - from lilbee.core.config import cfg - from lilbee.core.config.enums import KvCacheType - if cfg.kv_cache_type is KvCacheType.F16: return None return cfg.kv_cache_type.value @@ -451,7 +443,6 @@ def _chat_serve_budget_footprint(footprint: int) -> int: its context. Small models are unaffected -- they fit the serve budget with KV room to spare. """ - from lilbee.core.config import cfg return int(footprint * (USABLE_VRAM_FRACTION / cfg.gpu_memory_fraction)) @@ -465,7 +456,6 @@ def _placement_estimate_ctx(role: WorkerRole, model_path: Path, meta: dict[str, single-card placement) nor the full trained ceiling (which over-reserves). A model that cannot hold weights + this floor on one card is tensor-split. """ - from lilbee.core.config import cfg from lilbee.providers.engine_params import chat_ctx_ceiling if role is WorkerRole.CHAT: @@ -483,7 +473,6 @@ def _placement_estimate_slots(role: WorkerRole, meta: dict[str, str] | None) -> A tensor-split chat serves a single full-context sequence, so the placement total is the per-sequence ceiling, not ``ceiling x _CHAT_SLOTS`` (KV no launch allocates). """ - from lilbee.core.config import cfg if role is WorkerRole.CHAT: return _SPLIT_CHAT_SLOTS @@ -587,7 +576,6 @@ def _server_model_inputs( Skips an unconfigured optional role, a vision model with no resolvable mmproj projector, and a role whose model is not installed on disk. """ - from lilbee.core.config import cfg from lilbee.providers.base import ProviderError inputs: dict[WorkerRole, ModelPlacementInput] = {} @@ -654,7 +642,6 @@ def _launch_for( model_path = resolve_model_path(model_ref) weights_bytes = _weights_bytes(model_path) meta = read_gguf_metadata(model_path) - from lilbee.core.config import cfg chosen = tuple(by_index[i] for i in plan.devices) is_chat = plan.role is WorkerRole.CHAT @@ -897,7 +884,6 @@ def plan_launches( devices: list[FleetDevice], ) -> list[InstanceLaunch]: """Plan placement for *roles* (``None`` = all configured) and build their launches.""" - from lilbee.core.config import cfg unified_budget = _unified_memory_budget(devices) inputs, model_refs, reservation = _server_model_inputs( diff --git a/src/lilbee/providers/fleet/provider.py b/src/lilbee/providers/fleet/provider.py index 8165ecb0d..00baca6bc 100644 --- a/src/lilbee/providers/fleet/provider.py +++ b/src/lilbee/providers/fleet/provider.py @@ -248,7 +248,6 @@ def _vision_call( a ``ProviderError`` so the page-level OCR caller can fail just that page. Callers hold ``_VISION_GATE`` so queue time isn't billed against the timeout. """ - from lilbee.core.config import cfg options = {"max_tokens": cfg.vision_ocr_max_tokens} if timeout and timeout > 0: @@ -355,7 +354,6 @@ def _ensure_swap(self) -> SwapManager | None: # while this warm-up/reload thread was in flight; do not spawn a # llama-swap no live provider would ever reap. return None - from lilbee.core.config import cfg swap = SwapManager(cfg.data_dir) # A dead owner's surviving llama-swap holds VRAM; reap it before launching @@ -511,8 +509,6 @@ def _shutdown_swap(self, *, latch: bool = True) -> None: # never adopted, and SwapManager.shutdown reaps every llama-swap this # process spawned (keyed on our own children), not just a tracked handle. if swap is None: - from lilbee.core.config import cfg - swap = SwapManager(cfg.data_dir) swap.shutdown() @@ -600,7 +596,6 @@ def chat( the server parses native tool calls, so tool support needs no per-family parser here. """ - from lilbee.core.config import cfg from lilbee.providers.engine_params import chat_options_to_kwargs self._require_configured_model(model, str(cfg.chat_model), WorkerRole.CHAT) @@ -631,7 +626,6 @@ def chat_with_tools( model: str | None = None, ) -> ChatToolResult: """Route a tool-enabled chat turn to the least-busy chat server.""" - from lilbee.core.config import cfg from lilbee.providers.engine_params import chat_options_to_kwargs self._require_configured_model(model, str(cfg.chat_model), WorkerRole.CHAT) @@ -679,7 +673,6 @@ def embed(self, texts: list[str]) -> list[list[float]]: def vision_ocr( self, png_bytes: bytes, model: str, prompt: str = "", *, timeout: float | None = None ) -> str: - from lilbee.core.config import cfg from lilbee.vision import build_vision_messages, resolve_ocr_prompt self._require_configured_model(model, str(cfg.vision_model), WorkerRole.VISION) diff --git a/src/lilbee/providers/fleet/replicas.py b/src/lilbee/providers/fleet/replicas.py index 6dfa46dcc..b7273b379 100644 --- a/src/lilbee/providers/fleet/replicas.py +++ b/src/lilbee/providers/fleet/replicas.py @@ -11,6 +11,7 @@ import functools import logging +from lilbee.core.config import cfg from lilbee.providers.roles import ROLE_REGISTRY, WorkerRole log = logging.getLogger(__name__) @@ -26,7 +27,6 @@ def resolve_replica_count(role: WorkerRole, device_count: int) -> int: 0 means one replica per GPU (falling to one when GPU-less). Other roles run one instance. Capping to residual VRAM happens in placement. """ - from lilbee.core.config import cfg knob = ROLE_REGISTRY[role].replica_knob if knob is None: diff --git a/src/lilbee/providers/sdk_backend.py b/src/lilbee/providers/sdk_backend.py index 05149b7e2..0b1eb15c2 100644 --- a/src/lilbee/providers/sdk_backend.py +++ b/src/lilbee/providers/sdk_backend.py @@ -18,6 +18,7 @@ # Display name for the active backend the SDK is talking to. The # adapter's own identity is exposed separately via provider_name. +from lilbee.core.config import cfg from lilbee.providers.backend_names import BackendName from lilbee.providers.local_servers import detect_local_server @@ -56,7 +57,6 @@ def get_provider_api_key(provider: str) -> str | None: :data:`PROVIDER_API_KEY_FIELD`. Reads only the lilbee config field; use :func:`provider_has_key` to also honor the SDK's own env var. """ - from lilbee.core.config import cfg field = PROVIDER_API_KEY_FIELD.get(provider.lower()) if field is None: From c8f30837a01b2999ec09451fc4e5aa8c560e51a5 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:08:45 -0400 Subject: [PATCH 23/77] build(xberg): integrate 1.0.0rc5 cp310-abi3 wheel Bumps the local xberg trial wheel from rc2 to rc5 and relocks. rc5 is a clean drop-in: full suite green at 100% coverage with zero regression, abi3 wheel imports on 3.11 and 3.13, and the whole API surface lilbee uses is intact. rc5 also carries the tesseract ObjectCache teardown fix (kreuzberg #1190), so the earlier cosmetic-leak finding is resolved upstream. Still a local dev path until xberg publishes to an index. --- pyproject.toml | 4 ++-- uv.lock | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3114809f3..4d4d96c25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,9 +92,9 @@ packages = ["src/lilbee"] # the index instead -- this source override is workspace metadata, not published deps. [tool.uv.sources] lilbee-engine = { path = "packaging/engine-wheel" } -# TODO: xberg 1.0.0rc2 is a pre-publication trial wheel; resolve it from the local +# TODO: xberg 1.0.0rc5 is a pre-publication trial wheel; resolve it from the local # build until it lands on PyPI, then drop this source and lock from the index. -xberg = { path = "/Users/tobias/projects/kreuzberg-wt-pywheel/dist/xberg-1.0.0rc2-cp310-abi3-macosx_11_0_arm64.whl" } +xberg = { path = "/Users/tobias/projects/kreuzberg-wt-pywheel/dist/xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl" } [dependency-groups] dev = [ diff --git a/uv.lock b/uv.lock index 4f0f46fe2..1f58ba04d 100644 --- a/uv.lock +++ b/uv.lock @@ -1709,7 +1709,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "typing-extensions", specifier = ">=4.5.0" }, { name = "uvicorn", specifier = ">=0.30" }, - { name = "xberg", path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc2-cp310-abi3-macosx_11_0_arm64.whl" }, + { name = "xberg", path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl" }, ] provides-extras = ["litellm", "graph", "crawler", "cuda12", "release"] @@ -4450,10 +4450,10 @@ wheels = [ [[package]] name = "xberg" -version = "1.0.0rc2" -source = { path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc2-cp310-abi3-macosx_11_0_arm64.whl" } +version = "1.0.0rc5" +source = { path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:50577b30ceec53e4fcda990387a742066bb67722928590b661c70a1d221ad7fd" }, + { filename = "xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7854cde4a22de24ee03ff19b8b43ec6af84dfa30f93f5e2684bd18763c22ba62" }, ] [[package]] From 1485dde74c4c0f4f746288fab5c863b1c86d1098 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:14:31 -0400 Subject: [PATCH 24/77] build: add Intel-Mac (macos-15-intel) standalone build cells Brings the native Intel-Mac standalone build to this branch, mirroring the recipe on feat/intel-mac-support: two soft macos-15-intel cells (stock AVX lilbee-macos-x86_64 and a pre-Haswell lilbee-compat-macos-x86_64 held to x86-64-v2), the macOS 11.0 deployment-target pin on the Nuitka step so the launcher runs on macOS 11-14, and the x86_64 download entries on the site and in the README compat row. The Intel cells get their own install step because uv sync can't run there: stock lancedb 0.33.0 has no x86_64 macOS wheel and xberg is pinned to a local arm64 dev wheel. Both are pulled from the +compat index instead. This is gated on xberg publishing an x86_64 macOS wheel to that index -- until then the cell fails at install, but it is soft so it never blocks the release fan-out. --- .github/workflows/release.yml | 50 ++++++++++++++++++++++++++++++++++- README.md | 2 +- site/index.html | 25 +++++++++++++++++- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c7172debc..21f5ebff8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -136,7 +136,25 @@ jobs: - os: macos-latest asset_name: lilbee-macos-arm64 backend: metal - # Intel Mac dropped: macos-13 hosted runners stay queued; pip wheel covers them. + # Intel Mac: built natively on macos-15-intel, GitHub's Intel image + # (Nuitka can't cross-compile arch from the arm64 runner). Soft because + # the Intel image can queue longer than the arm64 pool -- this asset must + # never block the release fan-out. Deployment target stays 11.0, so the + # AVX baseline covers every Mac that can run macOS 11+ (Ivy Bridge or newer). + - os: macos-15-intel + asset_name: lilbee-macos-x86_64 + backend: cpu + soft: true + # Pre-Haswell-safe Intel Mac build: same self-contained model as the + # linux/windows compat cells (stock lancedb swapped for +compat, launcher + # held to x86-64-v2). Covers the AVX-but-no-AVX2 tail (e.g. 2013 Mac Pro) + # and guards against an AVX2-baselined stock lancedb SIGILL. + - os: macos-15-intel + asset_name: lilbee-compat-macos-x86_64 + backend: cpu + compat: true + soft: true + march: x86-64-v2 - os: windows-latest asset_name: lilbee-windows-x86_64.exe backend: vulkan @@ -218,8 +236,33 @@ jobs: save: ${{ github.ref_type != 'tag' }} - name: Install dependencies + if: matrix.os != 'macos-15-intel' run: uv sync --extra release && uv pip install 'nuitka[onefile]>=4.0,<5' pip + # Intel Mac: `uv sync` can't run here for two reasons -- stock lancedb 0.33.0 + # ships no macosx_x86_64 wheel, and xberg is pinned to a local arm64 dev wheel + # via [tool.uv.sources]. Resolve both from the +compat index (PEP 440 drop-ins), + # then install the rest of [release] fresh from PyPI (every other dep has an + # x86_64 macOS wheel). CMAKE_ARGS + deployment target hold any throwaway + # llama-cpp-python sdist build at the cpu/x86_64/macOS-11 baseline; the dedicated + # step below builds the llama-cpp-python that actually ships. + # NOTE: gated on xberg landing an x86_64 macOS wheel on the +compat index; until + # then this cell is expected to fail at install (it is soft, so it never blocks + # the release fan-out). + - name: Install dependencies (Intel Mac) + if: matrix.os == 'macos-15-intel' + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: "11.0" + run: | + set -euxo pipefail + eval "$(BACKEND=cpu TARGET_ARCH=x86_64 tools/wheel-build/cmake_args.sh)" + export CMAKE_ARGS + uv venv + uv pip install --extra-index-url https://lilbee.sh/compat/ \ + lancedb==0.33.0+compat xberg==1.0.0rc5 + uv pip install '.[release]' 'nuitka[onefile]>=4.0,<5' pip + - name: Verify extras installed in build venv shell: bash # --no-sync is load-bearing: re-syncing reinstalls python-multipart and shadows the shim. @@ -252,6 +295,11 @@ jobs: env: ASSET_NAME: ${{ matrix.asset_name }} PRODUCT_VERSION: ${{ env.LILBEE_NUITKA_VERSION }} + # Pin Nuitka's clang output to macOS 11.0. Unset, clang defaults the + # deployment target to the build host (macOS 15 on the Intel image), which + # would stamp the launcher minos=15 and refuse to launch on macOS 11-14 even + # though every bundled dylib targets 11.0. No-op off macOS. + MACOSX_DEPLOYMENT_TARGET: "11.0" # compat cell only: hold Nuitka's C output to the pre-Haswell baseline so # the launcher layer (not just lancedb) runs on old silicon. Empty elsewhere. CFLAGS: ${{ matrix.march && format('-march={0}', matrix.march) || '' }} diff --git a/README.md b/README.md index 980efa610..36d411094 100644 --- a/README.md +++ b/README.md @@ -341,7 +341,7 @@ Pre-2013 Intel or pre-Zen AMD CPUs lack [AVX2](https://en.wikipedia.org/wiki/Adv | **Scoop** | `scoop install lilbee-compat` | | **Flatpak** | `flatpak install lilbee io.github.tobocop2.lilbee.compat` | | **Snap** | `curl -LO https://github.com/tobocop2/lilbee/releases/latest/download/lilbee-compat-linux-x86_64.snap && sudo snap install ./lilbee-compat-linux-x86_64.snap --dangerous --classic` | -| **Binary** | [`lilbee-compat-linux-x86_64`](https://github.com/tobocop2/lilbee/releases/latest) or [`lilbee-compat-windows-x86_64.exe`](https://github.com/tobocop2/lilbee/releases/latest) | +| **Binary** | [`lilbee-compat-linux-x86_64`](https://github.com/tobocop2/lilbee/releases/latest), [`lilbee-compat-windows-x86_64.exe`](https://github.com/tobocop2/lilbee/releases/latest), or [`lilbee-compat-macos-x86_64`](https://github.com/tobocop2/lilbee/releases/latest) (pre-AVX2 Intel Macs, e.g. the 2013 Mac Pro) | Same `lilbee` command after install. The crash is from [lancedb](https://lancedb.github.io/lancedb/)'s AVX2-compiled wheels; this build swaps in a [lancedb fork](https://github.com/tobocop2/lance) that picks instructions at runtime. A 👍 or comment on the upstream [lance PR](https://github.com/lance-format/lance/pull/6630) helps it land. diff --git a/site/index.html b/site/index.html index 0e2a67c48..0714c1946 100644 --- a/site/index.html +++ b/site/index.html @@ -549,6 +549,18 @@

install

+
+
+ macOS x86_64 + download lilbee-macos-x86_64 → +
+
+ $ + curl -L https://github.com/tobocop2/lilbee/releases/latest/download/lilbee-macos-x86_64 -o lilbee && chmod +x lilbee && xattr -d com.apple.quarantine ./lilbee + +
+
+
Windows x86_64 @@ -597,6 +609,17 @@

install

+
+
+ macOS x86_64 + download lilbee-compat-macos-x86_64 → +
+
+ $ + curl -L https://github.com/tobocop2/lilbee/releases/latest/download/lilbee-compat-macos-x86_64 -o lilbee && chmod +x lilbee && xattr -d com.apple.quarantine ./lilbee + +
+
Windows x86_64 @@ -611,7 +634,7 @@

install

From 371908e0bb828343571a8249256b2357907aa755 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:34:05 -0400 Subject: [PATCH 25/77] Drop xberg type:ignores: rc5 stub fix + full OcrBackend conformance Two xberg # type: ignore workarounds are no longer needed on rc5: - chunk.py: rc5 types EmbeddingConfig.model as EmbeddingModelType (matching what .plugin() returns), so the arg-type ignore on EmbeddingConfig(model=...) is dead. Removed. - services.py: VisionOcrBackend now actually implements xberg's OcrBackend Protocol instead of registering a partial backend under an ignore. Per the Rust trait reference, added process_image_file (read + delegate), a document-processing stub, and the capability flags (all image-only defaults), and backend_type now returns OcrBackendType.CUSTOM. Config params are typed against the native OcrConfig the Protocol and runtime callbacks use. Two upstream xberg findings filed (kreuzberg-u4r, kreuzberg-50q): the public OcrConfig differs from the Protocol's native one, and the Python Protocol requires methods the Rust trait defaults. Full suite green at 100% coverage. --- src/lilbee/app/services.py | 4 +- src/lilbee/data/chunk.py | 5 +-- src/lilbee/data/ingest/vision_ocr_backend.py | 40 ++++++++++++++++++-- tests/test_vision_ocr_backend.py | 26 ++++++++++++- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/src/lilbee/app/services.py b/src/lilbee/app/services.py index 455e5d3e3..b7a109716 100644 --- a/src/lilbee/app/services.py +++ b/src/lilbee/app/services.py @@ -289,10 +289,8 @@ def sync_vision_ocr_backend(provider: LLMProvider) -> None: # Re-register so the backend always binds to the current provider. if registered: unregister_ocr_backend(OcrBackendName.LILBEE_VISION) - # xberg's OcrBackend Protocol lists file/document methods an image-only - # backend never needs; registration is duck-typed and validated at runtime. register_ocr_backend( - VisionOcrBackend(ocr_fn=provider.vision_ocr, model_ref_fn=lambda: cfg.vision_model) # type: ignore[arg-type] + VisionOcrBackend(ocr_fn=provider.vision_ocr, model_ref_fn=lambda: cfg.vision_model) ) elif registered: unregister_ocr_backend(OcrBackendName.LILBEE_VISION) diff --git a/src/lilbee/data/chunk.py b/src/lilbee/data/chunk.py index b8c0697ba..d2019c486 100644 --- a/src/lilbee/data/chunk.py +++ b/src/lilbee/data/chunk.py @@ -34,10 +34,7 @@ def _semantic_embedding_config() -> EmbeddingConfig: from lilbee.data.ingest.types import EmbeddingBackendName model = EmbeddingModelType.plugin(EmbeddingBackendName.LILBEE) - # xberg's public EmbeddingConfig still types `model` as the legacy - # str|int|LlmConfig alias, not the EmbeddingModelType class its own .plugin() - # returns; the constructor accepts the class instance at runtime. - return EmbeddingConfig(model=model) # type: ignore[arg-type] + return EmbeddingConfig(model=model) def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py index 2ebbf430e..08a8f369b 100644 --- a/src/lilbee/data/ingest/vision_ocr_backend.py +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -7,6 +7,7 @@ from contextlib import contextmanager from dataclasses import dataclass from importlib.metadata import PackageNotFoundError, version +from pathlib import Path from typing import TYPE_CHECKING, Any, Protocol from lilbee.data.ingest.types import MARKDOWN_MIME, OcrBackendName @@ -15,7 +16,13 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator - from xberg import ExtractedDocument, OcrConfig + from xberg import ExtractedDocument, OcrBackendType + + # The OcrBackend Protocol and the runtime trait callbacks use the native + # OcrConfig (xberg._xberg.OcrConfig), which is a different type from the public + # xberg.OcrConfig re-exported from xberg.options; a backend typed against the + # public one cannot satisfy the Protocol. Filed upstream (xberg). + from xberg._xberg import OcrConfig # Token key inside OcrConfig.backend_options JSON. xberg does not propagate # contextvars into process_image (xberg-4w9), so per-request state travels as @@ -134,8 +141,23 @@ def initialize(self) -> None: ... def shutdown(self) -> None: ... - def backend_type(self) -> str: - return "custom" + def backend_type(self) -> OcrBackendType: + from xberg import OcrBackendType + + return OcrBackendType.CUSTOM + + def supports_table_detection(self) -> bool: + return False + + def supports_document_processing(self) -> bool: + return False + + def emits_structured_markdown(self) -> bool: + # Keep xberg's default: the pre-migration differential was validated with + # layout reconstruction on. The vision model does emit markdown, so flipping + # this to True (skip reconstruction) is a valid optimization, but it changes + # real OCR output and must be validated on the live integration first. + return False def process_image(self, image_bytes: bytes, config: OcrConfig) -> ExtractedDocument: # xberg passes the OcrConfig as a native object and expects a native @@ -151,3 +173,15 @@ def process_image(self, image_bytes: bytes, config: OcrConfig) -> ExtractedDocum if ctx is not None and ctx.on_page is not None: ctx.on_page() return ExtractedDocument(content=text, mime_type=MARKDOWN_MIME) + + def process_image_file(self, path: str, config: OcrConfig) -> ExtractedDocument: + # OCR an image file by reading its bytes and delegating to process_image + # (mirrors xberg's default OcrBackend::process_image_file). + return self.process_image(Path(path).read_bytes(), config) + + def process_document(self, _path: str, _config: OcrConfig) -> ExtractedDocument: + # Image-only backend: xberg only calls this when supports_document_processing() + # is True, which it never is here, so document-level OCR is unreachable. + raise NotImplementedError( + "Document-level OCR is not supported by the lilbee vision backend" + ) diff --git a/tests/test_vision_ocr_backend.py b/tests/test_vision_ocr_backend.py index adbe2b722..a5be99efc 100644 --- a/tests/test_vision_ocr_backend.py +++ b/tests/test_vision_ocr_backend.py @@ -39,8 +39,10 @@ def test_name_is_enum_value(self): assert be.name() == OcrBackendName.LILBEE_VISION == "lilbee-vision" def test_backend_type_is_custom(self): + from xberg import OcrBackendType + be, _ = _backend() - assert be.backend_type() == "custom" + assert be.backend_type() == OcrBackendType.CUSTOM def test_supports_all_languages(self): be, _ = _backend() @@ -69,6 +71,28 @@ def test_initialize_shutdown_noop(self): assert be.initialize() is None assert be.shutdown() is None + def test_capability_flags_are_image_only(self): + be, _ = _backend() + assert be.supports_table_detection() is False + assert be.supports_document_processing() is False + assert be.emits_structured_markdown() is False + + def test_process_image_file_reads_bytes_and_delegates(self, tmp_path): + be, calls = _backend() + img = tmp_path / "scan.png" + img.write_bytes(b"PNG-on-disk") + out = be.process_image_file(str(img), _cfg()) + assert out.content == "# extracted" + assert out.mime_type == MARKDOWN_MIME + assert calls[0][0] == b"PNG-on-disk" + + def test_process_document_is_unsupported(self): + import pytest + + be, _ = _backend() + with pytest.raises(NotImplementedError): + be.process_document("/tmp/doc.pdf", _cfg()) + class TestProcessImage: def test_returns_extracted_document(self): From 5cbf7d9d8ab464c3a291841a1bc5096d94aa5b04 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:52:24 -0400 Subject: [PATCH 26/77] build(xberg): relock rc5 wheel hash after rebuild The local rc5 dev wheel was deleted in a disk cleanup and rebuilt; Rust builds aren't byte-reproducible, so the new wheel hashes differently. Update the local [tool.uv.sources] pin's locked hash so uv sync resolves. Ephemeral until xberg ships to an index and the local pin is dropped. --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 1f58ba04d..086d80482 100644 --- a/uv.lock +++ b/uv.lock @@ -4453,7 +4453,7 @@ name = "xberg" version = "1.0.0rc5" source = { path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7854cde4a22de24ee03ff19b8b43ec6af84dfa30f93f5e2684bd18763c22ba62" }, + { filename = "xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:03e5b4525e1366e11de7b42beea386405f6126e6be0b387411b0b3ef036b1d51" }, ] [[package]] From b8bda2c3b95ff622a70d2b1cfba507df98eab015 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 4 Jul 2026 03:16:04 -0400 Subject: [PATCH 27/77] build(xberg): integrate wheel built from main (rc5 + 3 commits) Rebuild the local trial wheel from xberg main (kreuzberg e702310938 = v1.0.0-rc.5 + 3 commits: the ORT execution-provider env overrides and the resolve_batch_concurrency dead-code lint gate) instead of the rc5 tag. The upstream version string is still 1.0.0rc5 until rc6 is cut, so the wheel filename and dependency constraint are unchanged; only the pin comment (source commit) and the locked hash move. Full suite green at 100% coverage. --- pyproject.toml | 6 ++++-- uv.lock | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4d4d96c25..a0162411f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,8 +92,10 @@ packages = ["src/lilbee"] # the index instead -- this source override is workspace metadata, not published deps. [tool.uv.sources] lilbee-engine = { path = "packaging/engine-wheel" } -# TODO: xberg 1.0.0rc5 is a pre-publication trial wheel; resolve it from the local -# build until it lands on PyPI, then drop this source and lock from the index. +# TODO: pre-publication trial wheel; resolve it from the local build until it +# lands on PyPI, then drop this source and lock from the index. Built from xberg +# main (kreuzberg e702310938 = v1.0.0-rc.5 + 3 commits); the version string is +# still 1.0.0rc5 upstream until rc6 is cut, so the filename is unchanged. xberg = { path = "/Users/tobias/projects/kreuzberg-wt-pywheel/dist/xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl" } [dependency-groups] diff --git a/uv.lock b/uv.lock index 086d80482..fa6b84fb3 100644 --- a/uv.lock +++ b/uv.lock @@ -4453,7 +4453,7 @@ name = "xberg" version = "1.0.0rc5" source = { path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:03e5b4525e1366e11de7b42beea386405f6126e6be0b387411b0b3ef036b1d51" }, + { filename = "xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:9748511a3494bd0ba4d6546178d536058b6a4b417934340e2d9fe1c7f4994d24" }, ] [[package]] From 596d5b76419f2ac23fbdd7908c26a0feb10830e0 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 4 Jul 2026 03:40:15 -0400 Subject: [PATCH 28/77] test(integration): migrate PDF OCR test off the removed extract_file API test_pdf_integration.py still imported xberg's pre-migration extract_file and read result.content off the raw result. xberg 1.x has no extract_file (extraction is extract(ExtractInput, config) -> ExtractionResult). Because tests/integration is pytest-ignored by default, this never ran and slipped through the migration. Route the tesseract and vision cases through lilbee's own aextract_document bridge (bytes + mime_type + filename), which unwraps ExtractionResult to the ExtractedDocument, and pass language=["eng"] to the tesseract OcrConfig to match how lilbee actually configures tesseract. The two tesseract tests now pass against real xberg. --- tests/integration/test_pdf_integration.py | 46 +++++++++++++++++------ 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_pdf_integration.py b/tests/integration/test_pdf_integration.py index dbff8fc8d..205a4d60d 100644 --- a/tests/integration/test_pdf_integration.py +++ b/tests/integration/test_pdf_integration.py @@ -132,11 +132,20 @@ class TestTesseractOcrFallback: ) async def test_tesseract_extracts_text(self): """Tesseract OCR produces non-empty text from the scanned PDF fixture.""" - from xberg import ExtractionConfig, OcrConfig, extract_file + from xberg import ExtractionConfig, OcrConfig - config = ExtractionConfig(ocr=OcrConfig(backend="tesseract"), force_ocr=True) - result = await extract_file(str(SCANNED_PDF), config=config) - assert len(result.content.strip()) > 0, "Tesseract produced empty text" + from lilbee.data.xberg_extract import aextract_document + + config = ExtractionConfig( + ocr=OcrConfig(backend="tesseract", language=["eng"]), force_ocr=True + ) + doc = await aextract_document( + SCANNED_PDF.read_bytes(), + mime_type="application/pdf", + filename=SCANNED_PDF.name, + config=config, + ) + assert len(doc.content.strip()) > 0, "Tesseract produced empty text" @pytest.mark.skipif( not shutil.which("tesseract"), @@ -144,11 +153,20 @@ async def test_tesseract_extracts_text(self): ) async def test_tesseract_extracts_known_phrases(self): """Tesseract OCR captures key phrases from the scanned document.""" - from xberg import ExtractionConfig, OcrConfig, extract_file + from xberg import ExtractionConfig, OcrConfig + + from lilbee.data.xberg_extract import aextract_document - config = ExtractionConfig(ocr=OcrConfig(backend="tesseract"), force_ocr=True) - result = await extract_file(str(SCANNED_PDF), config=config) - text_lower = result.content.lower() + config = ExtractionConfig( + ocr=OcrConfig(backend="tesseract", language=["eng"]), force_ocr=True + ) + doc = await aextract_document( + SCANNED_PDF.read_bytes(), + mime_type="application/pdf", + filename=SCANNED_PDF.name, + config=config, + ) + text_lower = doc.content.lower() # At least some of the rendered text should be recognized recognized = any( phrase in text_lower @@ -174,17 +192,23 @@ class TestVisionOcrFallback: """Vision OCR through the registered lilbee-vision xberg backend.""" async def _vision_extract(self) -> str: - from xberg import ExtractionConfig, OcrConfig, extract_file + from xberg import ExtractionConfig, OcrConfig from lilbee.app.services import get_services, sync_vision_ocr_backend from lilbee.data.ingest.types import OcrBackendName + from lilbee.data.xberg_extract import aextract_document sync_vision_ocr_backend(get_services().provider) config = ExtractionConfig( ocr=OcrConfig(backend=OcrBackendName.LILBEE_VISION), force_ocr=True ) - result = await extract_file(str(SCANNED_PDF), config=config) - return result.content + doc = await aextract_document( + SCANNED_PDF.read_bytes(), + mime_type="application/pdf", + filename=SCANNED_PDF.name, + config=config, + ) + return doc.content @pytest.mark.skipif( not _vision_model_available(), From f616df0eb299ec5b087bd6f5e3c9a404dc18b27f Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 4 Jul 2026 13:06:42 -0400 Subject: [PATCH 29/77] build(xberg): integrate 1.0.0rc6 (bindings regenerated off alef main) Bump the local trial wheel to xberg 1.0.0rc6 (kreuzberg e94bc15621), with the python bindings regenerated off alef main (2c3803e: builtins-qualified stubs + let_unit_value), atop rc6's binding-contract fixes. Pure binding changes; no xberg runtime difference (result shapes, page attribution, native-before-vision OCR, verify_excerpt all verified identical to rc5). rc6 resolves both upstream findings filed during earlier integration: - kreuzberg-u4r: the OcrBackend Protocol now types config as the public options.OcrConfig, so VisionOcrBackend conforms against the public OcrConfig with no type:ignore and no reach into xberg._xberg (mypy clean). - kreuzberg-50q: the Protocol now requires only process_image, supports_language, and backend_type; the rest are optional. Full suite green at 100% coverage. --- pyproject.toml | 8 +++++--- src/lilbee/data/ingest/vision_ocr_backend.py | 18 ++++++------------ uv.lock | 8 ++++---- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a0162411f..089d116f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,9 +94,11 @@ packages = ["src/lilbee"] lilbee-engine = { path = "packaging/engine-wheel" } # TODO: pre-publication trial wheel; resolve it from the local build until it # lands on PyPI, then drop this source and lock from the index. Built from xberg -# main (kreuzberg e702310938 = v1.0.0-rc.5 + 3 commits); the version string is -# still 1.0.0rc5 upstream until rc6 is cut, so the filename is unchanged. -xberg = { path = "/Users/tobias/projects/kreuzberg-wt-pywheel/dist/xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl" } +# 1.0.0rc6 (kreuzberg e702310938... -> e94bc15621) with the python bindings +# regenerated off alef main (2c3803e: builtins-qualified stubs + let_unit_value, +# atop the public-OcrConfig-in-Protocol contract fix). Pure binding fixes; no +# xberg runtime change vs the committed rc6 bindings. +xberg = { path = "/Users/tobias/projects/kreuzberg/dist/xberg-1.0.0rc6-cp310-abi3-macosx_11_0_arm64.whl" } [dependency-groups] dev = [ diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py index 774ac4e93..2dbcc8619 100644 --- a/src/lilbee/data/ingest/vision_ocr_backend.py +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -16,13 +16,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator - from xberg import ExtractedDocument, OcrBackendType - - # The OcrBackend Protocol and the runtime trait callbacks use the native - # OcrConfig (xberg._xberg.OcrConfig), which is a different type from the public - # xberg.OcrConfig re-exported from xberg.options; a backend typed against the - # public one cannot satisfy the Protocol. Filed upstream (xberg). - from xberg._xberg import OcrConfig + from xberg import ExtractedDocument, OcrBackendType, OcrConfig # Token key inside OcrConfig.backend_options JSON. xberg does not propagate # contextvars into process_image (xberg-4w9), so per-request state travels as @@ -85,8 +79,8 @@ def backend_options_for(token: str) -> dict[str, str]: class _OcrConfigView: """Typed reader over the xberg OcrConfig object passed to process_image. - xberg hands the callback a native OcrConfig (alef typed trait callbacks), so - its fields are read directly as attributes; ``backend_options`` is a dict. + xberg hands the callback the public ``xberg.OcrConfig`` dataclass, so its + fields are read directly as attributes; ``backend_options`` is a dict. """ def __init__(self, config: OcrConfig) -> None: @@ -160,9 +154,9 @@ def emits_structured_markdown(self) -> bool: return False def process_image(self, image_bytes: bytes, config: OcrConfig) -> ExtractedDocument: - # xberg passes the OcrConfig as a native object and expects a native - # ExtractedDocument back (alef typed trait callbacks); read the request - # token and prompt override off the config, return the OCR text as markdown. + # xberg passes the public OcrConfig dataclass and expects a native + # ExtractedDocument back; read the request token and prompt override off + # the config, return the OCR text as markdown. from xberg import ExtractedDocument view = _OcrConfigView(config) diff --git a/uv.lock b/uv.lock index fa6b84fb3..6cdaf8191 100644 --- a/uv.lock +++ b/uv.lock @@ -1709,7 +1709,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "typing-extensions", specifier = ">=4.5.0" }, { name = "uvicorn", specifier = ">=0.30" }, - { name = "xberg", path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl" }, + { name = "xberg", path = "../../../../kreuzberg/dist/xberg-1.0.0rc6-cp310-abi3-macosx_11_0_arm64.whl" }, ] provides-extras = ["litellm", "graph", "crawler", "cuda12", "release"] @@ -4450,10 +4450,10 @@ wheels = [ [[package]] name = "xberg" -version = "1.0.0rc5" -source = { path = "../../../../kreuzberg-wt-pywheel/dist/xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl" } +version = "1.0.0rc6" +source = { path = "../../../../kreuzberg/dist/xberg-1.0.0rc6-cp310-abi3-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:9748511a3494bd0ba4d6546178d536058b6a4b417934340e2d9fe1c7f4994d24" }, + { filename = "xberg-1.0.0rc6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ed3a85dc002b894b47049554cf3641c409c7d448c55fda0be25364bfcdba0fbe" }, ] [[package]] From 4e8466beae49328d75e6eb595980a5093d06068a Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 4 Jul 2026 16:17:48 -0400 Subject: [PATCH 30/77] ingest: per-file extraction trace logging for diagnostics One parseable line per extracted file (source, type, elapsed_ms, pages, chunks, ocr_pages, vision attribution) on a dedicated logger, plus a vision line whenever OCR fires. LILBEE_INGEST_TRACE=1 enables it; LILBEE_INGEST_TRACE_FILE mirrors the stream to a file so the destination is independent of the host front-end (the TUI's root handler is WARNING+ and otherwise swallows the lines). --- src/lilbee/data/ingest/extract.py | 21 ++++++ src/lilbee/data/ingest/pipeline.py | 4 ++ src/lilbee/data/ingest/trace.py | 87 ++++++++++++++++++++++ tests/test_ingest_trace.py | 111 +++++++++++++++++++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 src/lilbee/data/ingest/trace.py create mode 100644 tests/test_ingest_trace.py diff --git a/src/lilbee/data/ingest/extract.py b/src/lilbee/data/ingest/extract.py index 933283b7b..e40f76a72 100644 --- a/src/lilbee/data/ingest/extract.py +++ b/src/lilbee/data/ingest/extract.py @@ -5,6 +5,7 @@ import contextvars import logging +import time from collections.abc import Generator, Sequence from contextlib import contextmanager from pathlib import Path @@ -14,6 +15,7 @@ from lilbee.core.config import active_config from lilbee.data.chunk import build_chunking_config, chunk_text from lilbee.data.ingest.offload import to_ingest_thread +from lilbee.data.ingest.trace import ExtractionTrace, trace_extraction, trace_log from lilbee.data.ingest.types import ( IMAGE_CONTENT_TYPE, MARKDOWN_OUTPUT, @@ -246,10 +248,29 @@ def _tick() -> None: ExtractEvent(file=source_name, page=page_seen, total_pages=0), ) + trace_log.debug("extract-start source=%r type=%s", source_name, content_type) + started = time.perf_counter() with ocr_request(on_page=_tick, timeout=_effective_ocr_timeout()) as token: config = extraction_config(content_type_to_mode(content_type), ocr_token=token) # xberg's extract is async; awaiting it keeps the OCR page loop off this thread. doc = await aextract_document(path.read_bytes(), filename=path.name, config=config) + elapsed = time.perf_counter() - started + + # One trace line per xberg extraction (filename, timing, counts, OCR pages), + # plus a dedicated vision line for scanned files -- the diagnostics an xberg + # author needs. Emitted for empty results too: a slow file that yields nothing + # is exactly the case worth surfacing. + trace_extraction( + ExtractionTrace( + source=source_name, + content_type=content_type, + elapsed_s=elapsed, + page_count=len(doc.pages or []) or len(doc.chunks or []), + chunk_count=len(doc.chunks or []), + ocr_pages=page_seen, + vision_configured=bool(active_config().vision_model), + ) + ) if not doc.chunks: if content_type in (PDF_CONTENT_TYPE, IMAGE_CONTENT_TYPE): diff --git a/src/lilbee/data/ingest/pipeline.py b/src/lilbee/data/ingest/pipeline.py index 621af7505..bc09c0e1e 100644 --- a/src/lilbee/data/ingest/pipeline.py +++ b/src/lilbee/data/ingest/pipeline.py @@ -33,6 +33,7 @@ write_skip_markers, write_skip_reasons, ) +from lilbee.data.ingest.trace import configure_from_env as configure_trace_from_env from lilbee.data.ingest.types import ( ChunkRecord, FileChangePlan, @@ -531,6 +532,9 @@ async def ingest_batch( ingesting new ones so the two operations are atomic per file. When *cancel* is set, pending files raise CancelledError before starting. """ + # Honor LILBEE_INGEST_TRACE once per batch: it raises the trace loggers above + # the default WARNING so per-file extraction lines actually surface. + configure_trace_from_env() semaphore = asyncio.Semaphore(_max_concurrent()) window = _max_concurrent() * _TASK_WINDOW_MULTIPLIER total_files = len(files_to_process) diff --git a/src/lilbee/data/ingest/trace.py b/src/lilbee/data/ingest/trace.py new file mode 100644 index 000000000..bce1e4fcf --- /dev/null +++ b/src/lilbee/data/ingest/trace.py @@ -0,0 +1,87 @@ +"""Structured per-file extraction tracing, for sharing ingest diagnostics. + +Every xberg extraction emits one machine-parseable line on the ``lilbee.ingest.trace`` +logger: filename, wall-clock, chunk and page counts, and how many pages fell through +to OCR. Files that needed the vision model also emit a line on ``lilbee.ingest.vision`` +so ``grep vision`` yields exactly the set of scanned files. Enable with +``LILBEE_INGEST_TRACE=1`` (sets both loggers to DEBUG). +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from pathlib import Path + +trace_log = logging.getLogger("lilbee.ingest.trace") +vision_log = logging.getLogger("lilbee.ingest.vision") + +_TRACE_ENV = "LILBEE_INGEST_TRACE" + + +@dataclass(frozen=True) +class ExtractionTrace: + """One xberg extraction's measured outcome.""" + + source: str + content_type: str + elapsed_s: float + page_count: int + chunk_count: int + ocr_pages: int + vision_configured: bool + + @property + def used_vision(self) -> bool: + """A page fell through to OCR and the OCR backend is the vision model.""" + return self.ocr_pages > 0 and self.vision_configured + + def as_line(self) -> str: + """A stable key=value line, easy to grep, diff, and hand to the xberg author.""" + return ( + f"extract source={self.source!r} type={self.content_type} " + f"elapsed_ms={self.elapsed_s * 1000:.0f} pages={self.page_count} " + f"chunks={self.chunk_count} ocr_pages={self.ocr_pages} " + f"vision={'yes' if self.used_vision else 'no'}" + ) + + +def configure_from_env() -> None: + """Enable trace/vision logging per LILBEE_INGEST_TRACE; mirror to a file when + LILBEE_INGEST_TRACE_FILE is set. + + The file handler exists because host apps own the root handlers: the TUI + logs WARNING+ to its file, so INFO trace lines vanish there even with the + loggers enabled. A dedicated handler on these two loggers makes the trace + destination independent of whichever front-end is running.""" + if os.environ.get("LILBEE_INGEST_TRACE", "").strip().lower() not in {"1", "true", "yes"}: + return + trace_log.setLevel(logging.DEBUG) + vision_log.setLevel(logging.INFO) + target = os.environ.get("LILBEE_INGEST_TRACE_FILE", "").strip() + if not target: + return + resolved = str(Path(target).absolute()) + for logger in (trace_log, vision_log): + if any( + isinstance(h, logging.FileHandler) and h.baseFilename == resolved + for h in logger.handlers + ): + continue + handler = logging.FileHandler(resolved) + handler.setLevel(logging.DEBUG) + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")) + logger.addHandler(handler) + + +def trace_extraction(trace: ExtractionTrace) -> None: + """Log one extraction's outcome, plus a dedicated line if it needed vision.""" + trace_log.info("%s", trace.as_line()) + if trace.used_vision: + vision_log.info( + "vision-ocr source=%r ocr_pages=%d elapsed_ms=%.0f", + trace.source, + trace.ocr_pages, + trace.elapsed_s * 1000, + ) diff --git a/tests/test_ingest_trace.py b/tests/test_ingest_trace.py new file mode 100644 index 000000000..225d64875 --- /dev/null +++ b/tests/test_ingest_trace.py @@ -0,0 +1,111 @@ +"""The extraction trace: one parseable line per file, a vision line when OCR fired.""" + +from __future__ import annotations + +import logging + +import pytest + +from lilbee.data.ingest.trace import ( + ExtractionTrace, + configure_from_env, + trace_extraction, + trace_log, + vision_log, +) + + +def _trace(**kw: object) -> ExtractionTrace: + base = { + "source": "docs/report.pdf", + "content_type": "application/pdf", + "elapsed_s": 1.234, + "page_count": 10, + "chunk_count": 42, + "ocr_pages": 0, + "vision_configured": True, + } + base.update(kw) + return ExtractionTrace(**base) # type: ignore[arg-type] + + +def test_line_carries_filename_timing_and_counts() -> None: + line = _trace().as_line() + assert "source='docs/report.pdf'" in line + assert "elapsed_ms=1234" in line + assert "pages=10" in line + assert "chunks=42" in line + assert "ocr_pages=0" in line + assert "vision=no" in line + + +def test_used_vision_requires_ocr_pages_and_a_configured_model() -> None: + assert _trace(ocr_pages=5, vision_configured=True).used_vision is True + assert _trace(ocr_pages=5, vision_configured=False).used_vision is False # tesseract + assert _trace(ocr_pages=0, vision_configured=True).used_vision is False # all native + + +def test_native_only_file_emits_no_vision_line(caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.INFO, logger="lilbee.ingest.vision") + caplog.set_level(logging.INFO, logger="lilbee.ingest.trace") + trace_extraction(_trace(ocr_pages=0)) + assert any(r.name == "lilbee.ingest.trace" for r in caplog.records) + assert not any(r.name == "lilbee.ingest.vision" for r in caplog.records) + + +def test_scanned_file_emits_a_dedicated_vision_line(caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.INFO, logger="lilbee.ingest.vision") + trace_extraction(_trace(ocr_pages=7, vision_configured=True)) + vision_records = [r for r in caplog.records if r.name == "lilbee.ingest.vision"] + assert len(vision_records) == 1 + assert "ocr_pages=7" in vision_records[0].getMessage() + assert "docs/report.pdf" in vision_records[0].getMessage() + + +def test_env_flag_enables_both_loggers(monkeypatch: pytest.MonkeyPatch) -> None: + trace_log.setLevel(logging.WARNING) + vision_log.setLevel(logging.WARNING) + monkeypatch.setenv("LILBEE_INGEST_TRACE", "1") + configure_from_env() + assert trace_log.level == logging.DEBUG # includes the extract-start debug line + assert vision_log.level == logging.INFO # vision lines are INFO, and emit + + +def test_trace_file_receives_records_when_host_handlers_filter( + monkeypatch: pytest.MonkeyPatch, tmp_path: object +) -> None: + # The TUI's root file handler is WARNING+; the dedicated trace file must + # still receive every INFO line or the extraction report starves. + from pathlib import Path + + target = Path(str(tmp_path)) / "trace.log" + monkeypatch.setenv("LILBEE_INGEST_TRACE", "1") + monkeypatch.setenv("LILBEE_INGEST_TRACE_FILE", str(target)) + configure_from_env() + try: + trace_extraction(_trace(ocr_pages=3, vision_configured=True)) + for handler in [*trace_log.handlers, *vision_log.handlers]: + handler.flush() + text = target.read_text() + assert "extract source=" in text and "vision-ocr source=" in text + configure_from_env() # idempotent: same file re-configured adds no dup handler + resolved = str(target.absolute()) + dups = [h for h in trace_log.handlers if getattr(h, "baseFilename", "") == resolved] + assert len(dups) == 1 + finally: + for logger in (trace_log, vision_log): + for handler in list(logger.handlers): + logger.removeHandler(handler) + handler.close() + + +def test_trace_surfaces_under_a_warning_root(monkeypatch: pytest.MonkeyPatch) -> None: + # The real failure mode: root at WARNING (lilbee's default) silently drops + # INFO trace lines. The env flag must lift the named loggers so records emit. + logging.getLogger().setLevel(logging.WARNING) + trace_log.setLevel(logging.WARNING) + vision_log.setLevel(logging.WARNING) + monkeypatch.setenv("LILBEE_INGEST_TRACE", "1") + configure_from_env() + assert trace_log.isEnabledFor(logging.INFO) + assert vision_log.isEnabledFor(logging.INFO) From a95a325828d6aa37806d874f50e4700309c3b02c Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 4 Jul 2026 16:17:48 -0400 Subject: [PATCH 31/77] fix(ingest): resolve the OCR request token from JSON-string backend_options The native round-trip serializes OcrConfig.backend_options to a JSON string, so the isinstance(dict) guard silently dropped the token on every real OCR call: no per-page progress and the configured ocr_timeout never applied. Accept both shapes; malformed strings still resolve to no token. --- src/lilbee/data/ingest/vision_ocr_backend.py | 28 +++++++++++++++----- tests/test_vision_ocr_backend.py | 16 +++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py index 2dbcc8619..8cf83232b 100644 --- a/src/lilbee/data/ingest/vision_ocr_backend.py +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import threading import uuid from contextlib import contextmanager @@ -16,7 +17,13 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator - from xberg import ExtractedDocument, OcrBackendType, OcrConfig + from xberg import ExtractedDocument, OcrBackendType + + # The OcrBackend Protocol and the runtime trait callbacks use the native + # OcrConfig (xberg._xberg.OcrConfig), which is a different type from the public + # xberg.OcrConfig re-exported from xberg.options; a backend typed against the + # public one cannot satisfy the Protocol. Filed upstream (xberg). + from xberg._xberg import OcrConfig # Token key inside OcrConfig.backend_options JSON. xberg does not propagate # contextvars into process_image (xberg-4w9), so per-request state travels as @@ -79,8 +86,8 @@ def backend_options_for(token: str) -> dict[str, str]: class _OcrConfigView: """Typed reader over the xberg OcrConfig object passed to process_image. - xberg hands the callback the public ``xberg.OcrConfig`` dataclass, so its - fields are read directly as attributes; ``backend_options`` is a dict. + xberg hands the callback a native OcrConfig (alef typed trait callbacks), so + its fields are read directly as attributes; ``backend_options`` is a dict. """ def __init__(self, config: OcrConfig) -> None: @@ -92,7 +99,16 @@ def vlm_prompt(self) -> str | None: @property def request_token(self) -> str | None: + # The native round-trip hands backend_options back as a JSON STRING, not + # the dict lilbee put in (alef serializes the map). Accept both shapes, + # or the token -- and with it on_page progress and the OCR timeout -- + # silently vanishes for every OCR call. options = self._config.backend_options + if isinstance(options, str): + try: + options = json.loads(options) + except json.JSONDecodeError: + return None token = options.get(_REQUEST_TOKEN_KEY) if isinstance(options, dict) else None return token if isinstance(token, str) else None @@ -154,9 +170,9 @@ def emits_structured_markdown(self) -> bool: return False def process_image(self, image_bytes: bytes, config: OcrConfig) -> ExtractedDocument: - # xberg passes the public OcrConfig dataclass and expects a native - # ExtractedDocument back; read the request token and prompt override off - # the config, return the OCR text as markdown. + # xberg passes the OcrConfig as a native object and expects a native + # ExtractedDocument back (alef typed trait callbacks); read the request + # token and prompt override off the config, return the OCR text as markdown. from xberg import ExtractedDocument view = _OcrConfigView(config) diff --git a/tests/test_vision_ocr_backend.py b/tests/test_vision_ocr_backend.py index a5be99efc..47f9bdf02 100644 --- a/tests/test_vision_ocr_backend.py +++ b/tests/test_vision_ocr_backend.py @@ -126,6 +126,22 @@ def test_request_context_supplies_timeout_and_fires_progress(self): assert calls[0][3] == 12.5 assert ticks == [1] + def test_json_string_backend_options_resolve(self): + # The native round-trip hands backend_options back as a JSON STRING + # (alef serializes the map). The token must resolve from that shape or + # per-page progress and the OCR timeout silently vanish on every real + # extraction, while dict-based unit tests stay green. + import json + + ticks: list[int] = [] + be, calls = _backend() + with ocr_request(on_page=lambda: ticks.append(1), timeout=7.5) as token: + be.process_image( + b"PNG", _cfg(backend_options=json.dumps(backend_options_for(token))) + ) + assert calls[0][3] == 7.5 + assert ticks == [1] + def test_no_context_uses_zero_timeout_and_no_tick(self): be, calls = _backend() be.process_image(b"PNG", _cfg(backend_options=backend_options_for("unknown-token"))) From 62a9b6a15ee6983635a9d91e9c27196f7460a33a Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 4 Jul 2026 16:17:48 -0400 Subject: [PATCH 32/77] fix(ingest): embed ctx = the chunker's character budget, the provable token ceiling chunk_size is token-denominated but the chunker enforces a CHARACTER budget (chunk_size * CHARS_PER_TOKEN). Token-dense text tokenizes near 1.5-2.5 chars/token, so real chunks approached 2000 tokens against a 512-token cap and lost their tails at embed time -- silently unsearchable text. A BPE token is at least one character, so the char budget is a hard upper bound on any chunk's token count: sizing the embed context to it (capped by trained context) makes embed-time truncation impossible. --- src/lilbee/providers/engine_params.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/lilbee/providers/engine_params.py b/src/lilbee/providers/engine_params.py index 55ca7bb03..212aed5b4 100644 --- a/src/lilbee/providers/engine_params.py +++ b/src/lilbee/providers/engine_params.py @@ -40,9 +40,19 @@ def resolve_embed_ctx(meta: dict[str, str] | None, model_path: Path) -> int: - """Embed/rerank context: chunk length plus truncation margin, capped by trained context.""" + """Embed/rerank context: worst-case chunk tokenization, capped by trained context. + + ``chunk_size`` is token-denominated but the chunker enforces a CHARACTER + budget (``chunk_size * CHARS_PER_TOKEN``). A BPE token is at least one + character, so that char budget is also the PROVABLE token ceiling for any + chunk the chunker can emit: size the context to it and embed-time + truncation becomes impossible, not merely rare. (Observed live before the + fix: numeric-table chunks at ~1.5 chars/token reached 1982 tokens against a + 2x-chunk_size cap and lost their tails -- silently unsearchable text.)""" + from lilbee.data.chunk import CHARS_PER_TOKEN + train_ctx = train_ctx_from_meta(meta, fallback=EMBED_FALLBACK_CTX, model_path=model_path) - return min(train_ctx, cfg.chunk_size + _EMBED_CTX_MARGIN) + return min(train_ctx, cfg.chunk_size * CHARS_PER_TOKEN + _EMBED_CTX_MARGIN) _LLM_RERANK_HEADROOM = 512 From c22d96e71b3094f67ba082cd740d9e983e4252f8 Mon Sep 17 00:00:00 2001 From: Tobias <5562156+tobocop2@users.noreply.github.com> Date: Sat, 4 Jul 2026 18:44:09 -0400 Subject: [PATCH 33/77] fix: resolve five local-model-api bugs (#482) bb-5fn: the wheel build pinned gguf-parser v0.9.3, whose JSON predates the estimate.items schema vram.py reads, so every role estimate raised and planning logged "model is not installed" for a model that was installed. Bump the pin to v0.24.1 and split the planning warning by ProviderError.kind so a sizing failure says "could not size model" instead of misdirecting toward the registry. bb-szm: /api/chat treated top_k:0 as "use the configured default" rather than "skip retrieval", so a pure-LLM call still grounded and returned sources. The request default is now None (unspecified -> configured default); an explicit 0 skips retrieval on both the one-shot and streaming paths. The ask endpoints are unchanged. bb-cpu: when reasoning consumed the whole generation, answer extraction produced an empty string and /api/chat returned answer:"" with 201, indistinguishable from a genuine empty response. Both the one-shot and streaming paths now surface a clear notice when reasoning ran but no final answer followed. bb-v3t: a client polling /api/health could not tell a fleet that was still loading from one that never started warming (both read chat_ready:false), which looked like a silent hang on a fresh box with no chat model. Add chat_status (ready / loading / not_started / error) alongside the existing bool. bb-z59: a bare org/repo ref to a raw-cache split GGUF resolved to shard 1's blob (siblings not co-located, so unloadable) and wrote a single-file manifest. Recovery is now shard-aware: it resolves the whole set from the first shard's snapshot symlink with full shard accounting. --- src/lilbee/modelhub/registry.py | 13 +++ src/lilbee/providers/fleet/planning.py | 22 ++-- src/lilbee/retrieval/reasoning.py | 10 ++ src/lilbee/server/handlers/__init__.py | 19 +++- src/lilbee/server/handlers/rag.py | 65 ++++++++---- src/lilbee/server/models.py | 12 ++- tests/test_fleet_planning.py | 28 +++++ tests/test_registry.py | 26 +++++ tests/test_server_handlers.py | 132 ++++++++++++++++++++++++ tools/wheel-build/build_llama_server.sh | 2 +- 10 files changed, 300 insertions(+), 29 deletions(-) diff --git a/src/lilbee/modelhub/registry.py b/src/lilbee/modelhub/registry.py index d1afa7a84..eb321adaa 100644 --- a/src/lilbee/modelhub/registry.py +++ b/src/lilbee/modelhub/registry.py @@ -279,6 +279,19 @@ def _resolve_repo_only(self, hf_repo: str) -> Path: if blob.exists(): return blob for filename in sorted(self._cached_gguf_names(hf_repo)): + shards = split_shard_filenames(filename) + if len(shards) > 1: + # A split set in the cache: skip its non-first shards and resolve + # the whole set from shard 1 so we hand back the snapshot symlink + # (siblings co-located, loadable) with shard accounting, not + # shard 1's blob as an unloadable single file. + if filename != shards[0]: + continue + with contextlib.suppress(KeyError): + return self._resolve_split( + format_native_gguf_ref(hf_repo, filename), hf_repo, shards + ) + continue recovered = self._find_cached_gguf(hf_repo, filename) if recovered is not None: self._reregister_from_cache(hf_repo, filename, recovered) diff --git a/src/lilbee/providers/fleet/planning.py b/src/lilbee/providers/fleet/planning.py index 0a547229c..5ab76840d 100644 --- a/src/lilbee/providers/fleet/planning.py +++ b/src/lilbee/providers/fleet/planning.py @@ -586,7 +586,7 @@ def _server_model_inputs( Skips an unconfigured optional role, a vision model with no resolvable mmproj projector, and a role whose model is not installed on disk. """ - from lilbee.providers.base import ProviderError + from lilbee.providers.base import ProviderError, ProviderErrorKind inputs: dict[WorkerRole, ModelPlacementInput] = {} model_refs: dict[WorkerRole, str] = {} @@ -609,13 +609,19 @@ def consider(role: WorkerRole, *, chat_reservation: int = 0) -> None: chat_reservation=chat_reservation, device_count=device_count, ) - except (ProviderError, OSError): - # The configured model is not installed/resolvable. Skip this role - # rather than failing the whole fleet build: search-only indexing - # must not require an installed chat model, and a genuinely-needed - # role surfaces a clear per-role error on first use instead of a - # build-time traceback. - log.warning("Skipping %s server: model %r is not installed.", role.value, ref) + except (ProviderError, OSError) as exc: + # Skip this role rather than failing the whole fleet build: search-only + # indexing must not require an installed chat model, and a genuinely- + # needed role surfaces a clear per-role error on first use instead of a + # build-time traceback. Keep "not installed" for an actually-missing + # model; a sizing failure (estimator errored) must say so, not misdirect + # debugging toward the registry. + if isinstance(exc, ProviderError) and exc.kind is ProviderErrorKind.NOT_FOUND: + log.warning("Skipping %s server: model %r is not installed.", role.value, ref) + else: + log.warning( + "Skipping %s server: could not size model %r (%s).", role.value, ref, exc + ) return inputs[role] = estimate model_refs[role] = ref diff --git a/src/lilbee/retrieval/reasoning.py b/src/lilbee/retrieval/reasoning.py index 36cc36786..8ed08b5b7 100644 --- a/src/lilbee/retrieval/reasoning.py +++ b/src/lilbee/retrieval/reasoning.py @@ -44,6 +44,16 @@ CAP_NOTICE_TEMPLATE = "\n[reasoning capped at {chars} chars, asking for a direct answer]\n" """User-visible marker emitted between the truncated reasoning and the continuation answer.""" +REASONING_EXHAUSTED_NOTICE = ( + "The model spent its whole response budget on reasoning and produced no final " + "answer. Try a shorter question, raise the generation token limit, or lower the " + "reasoning effort." +) +"""Returned in place of an empty answer when reasoning consumed the entire generation. + +Lets a caller tell "the model thought itself to death" apart from a genuine empty +response, which an empty string alone cannot.""" + @dataclass class StreamToken: diff --git a/src/lilbee/server/handlers/__init__.py b/src/lilbee/server/handlers/__init__.py index 3cb076418..8a5163a27 100644 --- a/src/lilbee/server/handlers/__init__.py +++ b/src/lilbee/server/handlers/__init__.py @@ -14,7 +14,7 @@ import dataclasses import time from collections.abc import AsyncGenerator, Sequence -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal from lilbee.app.services import get_services from lilbee.app.status import gather_status @@ -78,6 +78,7 @@ if TYPE_CHECKING: from lilbee.app.placement import PlacementView + from lilbee.providers.base import LLMProvider from lilbee.server.models import GpuInfoResponse, PlacementResponse # How often the warm stream re-snapshots provider state; sub-second so the read @@ -89,6 +90,21 @@ _WARM_STREAM_TIMEOUT_S = 1800.0 +def _chat_status(provider: LLMProvider) -> Literal["ready", "loading", "not_started", "error"]: + """Classify the chat engine's readiness for /api/health. + + ``ready`` once the role serves; ``error`` when warm-up failed; ``loading`` + while a warm is in flight; ``not_started`` when nothing is warming and the + role isn't up (no chat model planned, so the fleet won't come up on its own). + """ + if provider.role_ready(WorkerRole.CHAT): + return "ready" + snapshot = provider.warm_progress() + if snapshot is None: + return "not_started" + return "error" if snapshot.phase is WarmPhase.ERROR else "loading" + + async def health() -> HealthResponse: """Return service health, version, and whether the chat engine is warm.""" provider = get_services().provider @@ -96,6 +112,7 @@ async def health() -> HealthResponse: status="ok", version=get_version(), chat_ready=provider.role_ready(WorkerRole.CHAT), + chat_status=_chat_status(provider), chat_ctx=provider.served_chat_ctx(), ) diff --git a/src/lilbee/server/handlers/rag.py b/src/lilbee/server/handlers/rag.py index 17c0bb4dc..47fd8b8a5 100644 --- a/src/lilbee/server/handlers/rag.py +++ b/src/lilbee/server/handlers/rag.py @@ -23,6 +23,7 @@ from lilbee.retrieval.reasoning import ( CAP_CONTINUATION_PROMPT, CAP_NOTICE_TEMPLATE, + REASONING_EXHAUSTED_NOTICE, CapNotice, StreamToken, TagParser, @@ -64,6 +65,7 @@ if TYPE_CHECKING: from lilbee.core.results import SearchChunk from lilbee.retrieval.query import ChatMessage + from lilbee.retrieval.query.searcher import Searcher log = logging.getLogger(__name__) @@ -324,7 +326,7 @@ def ask_stream( async def chat( question: str, history: list[ChatMessage], - top_k: int = 0, + top_k: int | None = None, options: dict[str, Any] | None = None, chunk_type: ChunkType | None = None, ) -> AskResponse: @@ -339,6 +341,11 @@ async def chat( response = await asyncio.to_thread(dispatch_chat, req) text = _join_text_blocks(response.content) answer = text if cfg.show_reasoning else strip_reasoning(text) + if not answer.strip() and text.strip(): + # The model emitted only reasoning (stripped to nothing) and no final + # answer. Surface that distinctly instead of a silent empty string the + # caller can't tell apart from a legitimate empty response (bb-cpu). + answer = REASONING_EXHAUSTED_NOTICE await _store_extracted_memories(question, answer) return AskResponse( answer=answer, @@ -350,7 +357,7 @@ async def chat( def chat_stream( question: str, history: list[ChatMessage], - top_k: int = 0, + top_k: int | None = None, options: dict[str, Any] | None = None, chunk_type: ChunkType | None = None, ) -> AsyncGenerator[str, None]: @@ -363,7 +370,7 @@ def chat_stream( async def _stream_chat_response( question: str, history: list[ChatMessage], - top_k: int, + top_k: int | None, options: dict[str, Any] | None, chunk_type: ChunkType | None, ) -> AsyncGenerator[str, None]: @@ -379,14 +386,15 @@ async def _stream_chat_response( yield sse_event(SseEvent.SOURCES, []) yield sse_done({}) return - if searcher.skip_retrieval(): - # Chat-only mode (or no embedder): answer directly with no RAG context. + if _retrieval_off(searcher, top_k): + # Chat-only mode, no embedder, or an explicit top_k:0 pure-LLM call: + # answer directly with no RAG context. sources: list[SearchChunk] = [] messages = searcher.direct_messages(question, history) else: try: rag = searcher.build_rag_context( - question, top_k=top_k, history=history, chunk_type=chunk_type + question, top_k=top_k or 0, history=history, chunk_type=chunk_type ) except EmbeddingModelMismatchError as exc: # detail carries the index's embedder so the client can offer to adopt it. @@ -431,23 +439,32 @@ async def _cap_aware_chat_events( Mirrors :func:`stream_chat_with_cap` but consumes the canonical async stream. ``CapNotice`` is yielded once between the truncated reasoning and the continuation answer; ``StreamToken`` carries the - reasoning-vs-response split for downstream SSE shaping. + reasoning-vs-response split for downstream SSE shaping. When reasoning + runs but no final answer follows, a closing ``StreamToken`` carrying + ``REASONING_EXHAUSTED_NOTICE`` is yielded so the run isn't silent (bb-cpu). """ cap_chars = effective_reasoning_cap() show = cfg.show_reasoning + answered = False first_parser = TagParser(show=show) async for tok in _drive_stream(dispatch_chat_stream(req), first_parser, cap_chars): + answered = answered or (not tok.is_reasoning and bool(tok.content)) yield tok - if not (cap_chars > 0 and first_parser.reasoning_chars > cap_chars): - return - yield CapNotice(cap_chars=cap_chars) - nudged = _nudged_request(req) - cont_parser = TagParser(show=show) - async for tok in _drive_stream(dispatch_chat_stream(nudged), cont_parser, cap_chars=0): - # Continuation tokens are always treated as final-answer text. - yield StreamToken(content=tok.content, is_reasoning=False) + if cap_chars > 0 and first_parser.reasoning_chars > cap_chars: + yield CapNotice(cap_chars=cap_chars) + nudged = _nudged_request(req) + cont_parser = TagParser(show=show) + async for tok in _drive_stream(dispatch_chat_stream(nudged), cont_parser, cap_chars=0): + answered = answered or bool(tok.content) + # Continuation tokens are always treated as final-answer text. + yield StreamToken(content=tok.content, is_reasoning=False) + + if first_parser.reasoning_chars > 0 and not answered: + # The model spent its budget reasoning and produced no final answer; + # a distinct notice tells a reasoning-only run apart from a completed one. + yield StreamToken(content=REASONING_EXHAUSTED_NOTICE, is_reasoning=False) async def _drive_stream( @@ -513,10 +530,20 @@ def _text_from_event(event: Any) -> str: return "" +def _retrieval_off(searcher: Searcher, top_k: int | None) -> bool: + """Whether this /api/chat turn bypasses RAG. + + An explicit ``top_k == 0`` is a pure-LLM call: answer without retrieval. An + unspecified ``top_k`` (``None``) uses the configured default and grounds + normally. Chat-only mode or a missing embedder also bypass. + """ + return top_k == 0 or searcher.skip_retrieval() + + def _build_chat_messages( question: str, history: list[ChatMessage], - top_k: int, + top_k: int | None, chunk_type: ChunkType | None, ) -> tuple[list[SearchChunk], list[ChatMessage]]: """Run retrieval and return (sources, message_list). @@ -526,9 +553,11 @@ def _build_chat_messages( ``Searcher.build_rag_context``. """ searcher = get_services().searcher - if searcher.skip_retrieval(): + if _retrieval_off(searcher, top_k): return [], searcher.direct_messages(question, history) - rag = searcher.build_rag_context(question, top_k=top_k, history=history, chunk_type=chunk_type) + rag = searcher.build_rag_context( + question, top_k=top_k or 0, history=history, chunk_type=chunk_type + ) if rag is None: return [], searcher.direct_messages(question, history) return rag diff --git a/src/lilbee/server/models.py b/src/lilbee/server/models.py index 1e7f8e34c..8b4a76d34 100644 --- a/src/lilbee/server/models.py +++ b/src/lilbee/server/models.py @@ -52,7 +52,9 @@ class ChatRequest(BaseModel): question: str history: list[ChatMessage] = [] - top_k: int = Field(default=0, le=100) + # None (unspecified) grounds with the configured top_k; an explicit 0 is a + # pure-LLM call that skips retrieval entirely. + top_k: int | None = Field(default=None, le=100) options: dict[str, Any] | None = None chunk_type: ChunkType | None = None @@ -175,6 +177,14 @@ class HealthResponse(BaseModel): A launcher polls this to wait out the cold model load before handing off to a client, so the client never lands on an apparently-dead stream. """ + chat_status: Literal["ready", "loading", "not_started", "error"] = "not_started" + """Finer-grained chat readiness than the ``chat_ready`` bool. + + Lets a polling client tell a fleet that is still loading (wait) apart from one + that never started warming (``not_started`` -- no chat model resolved / planned, + so it will not come up on its own) or failed (``error``). Without this a bare + ``chat_ready:false`` reads the same for "loading" and "hung", which looked like a + silent hang on a fresh box with no chat model installed.""" chat_ctx: int | None = None """Per-slot context the chat engine serves, so a launcher can tell the client its window and the client trims history to fit. None until the engine is up.""" diff --git a/tests/test_fleet_planning.py b/tests/test_fleet_planning.py index ea3741a1c..afc92376f 100644 --- a/tests/test_fleet_planning.py +++ b/tests/test_fleet_planning.py @@ -657,6 +657,34 @@ def _estimate(role, ref, **_k): assert WorkerRole.CHAT not in refs assert {i.role for i in inputs} == {WorkerRole.EMBED} + def test_server_model_inputs_distinguishes_sizing_failure_from_missing( + self, monkeypatch, caplog + ) -> None: + # A sizing failure (estimator errored) must not be reported as "not + # installed" -- that misdirects debugging toward the registry when the + # real fault is the memory estimator. + import logging + + from lilbee.providers.base import ProviderError, ProviderErrorKind + + def _estimate(role, ref, **_k): + raise ProviderError( + "estimator returned no usable result", + provider="llama-server", + kind=ProviderErrorKind.SERVER, + ) + + monkeypatch.setattr(planning_mod, "_estimate_role", _estimate) + monkeypatch.setattr(cfg, "chat_model", "org/repo/chat.gguf") + monkeypatch.setattr(cfg, "embedding_model", "org/repo/embed.gguf") + monkeypatch.setattr(cfg, "reranker_model", "") + monkeypatch.setattr(cfg, "vision_model", "") + with caplog.at_level(logging.WARNING): + inputs, refs, _res = planning_mod._server_model_inputs() + assert not refs and not inputs + assert "is not installed" not in caplog.text + assert "could not size" in caplog.text + def test_server_model_inputs_includes_configured_rerank(self, monkeypatch) -> None: monkeypatch.setattr( planning_mod, "_estimate_role", lambda role, ref, **_k: ModelPlacementInput(role, _GB) diff --git a/tests/test_registry.py b/tests/test_registry.py index 1b227dcd7..9b77a1bb3 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -425,6 +425,32 @@ def test_split_gguf_resolves_to_snapshot_symlink_with_siblings(self, tmp_path: P for n in (2, 3): assert (resolved.parent / f"m-mxfp4-0000{n}-of-00003.gguf").exists() + def test_bare_repo_recovers_split_gguf_from_raw_cache(self, tmp_path: Path) -> None: + """bb-z59: a bare org/repo ref pointing at a raw-cache split GGUF (hf + download, no manifest) resolves to the first shard's snapshot symlink with + shard accounting, not shard 1's blob as an unloadable single file.""" + registry = ModelRegistry(tmp_path) + repo = "ggml-org/gpt-oss-120b-GGUF" + for n in (1, 2, 3): + _seed_hf_cache( + tmp_path, + repo=repo, + filename=f"m-mxfp4-0000{n}-of-00003.gguf", + content=f"shard-{n}".encode(), + ) + resolved = registry.resolve(repo) + # The bare-repo path lands on the same loadable snapshot symlink as the + # canonical first-shard ref, not a blob under blobs/. + assert resolved == registry.resolve(f"{repo}/m-mxfp4-00001-of-00003.gguf") + assert resolved.parent.name == _FAKE_REV + # Recovery wrote a manifest with full shard accounting, so it lists + # installed: shard_blobs holds the trailing shards (2 and 3); the first + # shard is the primary blob. + manifest = registry._read_manifest(repo, "m-mxfp4-00001-of-00003.gguf") + assert manifest is not None + assert len(manifest.shard_blobs) == 2 + assert manifest.total_size_bytes == sum(len(f"shard-{n}".encode()) for n in (1, 2, 3)) + def test_shard_paths_returns_every_split_shard(self, tmp_path: Path) -> None: registry = ModelRegistry(tmp_path) repo = "ggml-org/gpt-oss-120b-GGUF" diff --git a/tests/test_server_handlers.py b/tests/test_server_handlers.py index 0e83c67c1..b6cab1188 100644 --- a/tests/test_server_handlers.py +++ b/tests/test_server_handlers.py @@ -117,6 +117,25 @@ async def test_chat_ready_reflects_role_readiness(self, mock_svc): mock_svc.provider.role_ready.return_value = True assert (await handlers.health()).chat_ready is True + async def test_chat_status_distinguishes_not_started_from_loading(self, mock_svc): + """bb-v3t: a polling client must tell a fleet that never started warming + apart from one that is loading, so a fresh box with no chat model doesn't + look like a silent hang.""" + from lilbee.providers.warm_progress import WarmPhase, WarmProgress + + mock_svc.provider.role_ready.return_value = True + assert (await handlers.health()).chat_status == "ready" + + mock_svc.provider.role_ready.return_value = False + mock_svc.provider.warm_progress.return_value = None + assert (await handlers.health()).chat_status == "not_started" + + mock_svc.provider.warm_progress.return_value = WarmProgress(phase=WarmPhase.READING_WEIGHTS) + assert (await handlers.health()).chat_status == "loading" + + mock_svc.provider.warm_progress.return_value = WarmProgress(phase=WarmPhase.ERROR) + assert (await handlers.health()).chat_status == "error" + def _parse_warm_events(chunks: list[str]) -> list[dict]: """Pull the JSON payloads off ``warm`` SSE chunks, ignoring the [DONE] tail.""" @@ -707,6 +726,89 @@ def _fake_dispatch(req): assert result.sources == [] # direct-chat fallback carries no sources assert len(captured) == 1 + async def test_top_k_zero_skips_retrieval(self, mock_svc, monkeypatch): + """bb-szm: an explicit top_k:0 is a pure-LLM call -- retrieval is + bypassed and no sources are attached, even in search mode.""" + from lilbee.server.chat_dispatch.canonical import ( + CanonicalResponse, + CanonicalUsage, + StopReason, + TextBlock, + ) + + monkeypatch.setattr( + _rag_h, + "dispatch_chat", + lambda req: CanonicalResponse( + id="msg_test", + model=req.model, + content=[TextBlock(text="direct")], + stop_reason=StopReason.END_TURN, + usage=CanonicalUsage(input_tokens=0, output_tokens=0), + ), + ) + monkeypatch.setattr(cfg, "chat_mode", ChatMode.SEARCH.value) + mock_svc.searcher.direct_messages.return_value = [{"role": "user", "content": "q"}] + result = await handlers.chat("q", [], top_k=0) + mock_svc.searcher.build_rag_context.assert_not_called() + mock_svc.searcher.direct_messages.assert_called_once() + assert result.sources == [] + + async def test_top_k_none_grounds_normally(self, mock_svc, monkeypatch): + """Unspecified top_k (None) still runs retrieval -- only an explicit 0 skips.""" + from lilbee.server.chat_dispatch.canonical import ( + CanonicalResponse, + CanonicalUsage, + StopReason, + TextBlock, + ) + + monkeypatch.setattr( + _rag_h, + "dispatch_chat", + lambda req: CanonicalResponse( + id="msg_test", + model=req.model, + content=[TextBlock(text="ok")], + stop_reason=StopReason.END_TURN, + usage=CanonicalUsage(input_tokens=0, output_tokens=0), + ), + ) + monkeypatch.setattr(cfg, "chat_mode", ChatMode.SEARCH.value) + mock_svc.searcher.build_rag_context.return_value = _rag_return() + await handlers.chat("q", [], top_k=None) + mock_svc.searcher.build_rag_context.assert_called_once() + # None resolves to the config default (0 -> searcher picks cfg.top_k). + assert mock_svc.searcher.build_rag_context.call_args.kwargs.get("top_k") == 0 + + async def test_reasoning_only_answer_surfaces_notice(self, mock_svc, monkeypatch): + """bb-cpu: when reasoning consumes the whole generation and no final answer + follows, /chat returns a clear notice instead of a silent empty string.""" + from lilbee.retrieval.reasoning import REASONING_EXHAUSTED_NOTICE + from lilbee.server.chat_dispatch.canonical import ( + CanonicalResponse, + CanonicalUsage, + StopReason, + TextBlock, + ) + + monkeypatch.setattr( + _rag_h, + "dispatch_chat", + lambda req: CanonicalResponse( + id="msg_test", + model=req.model, + content=[TextBlock(text="a long chain of thought")], + stop_reason=StopReason.MAX_TOKENS, + usage=CanonicalUsage(input_tokens=0, output_tokens=0), + ), + ) + monkeypatch.setattr(cfg, "chat_mode", ChatMode.SEARCH.value) + monkeypatch.setattr(cfg, "show_reasoning", False) + mock_svc.searcher.build_rag_context.return_value = _rag_return() + result = await handlers.chat("q", []) + assert result.answer == REASONING_EXHAUSTED_NOTICE + class TestChatStream: async def test_no_results_yields_error(self, mock_svc): @@ -734,6 +836,36 @@ async def test_forwards_chunk_type_to_build_rag_context(self, mock_svc): pass assert mock_svc.searcher.build_rag_context.call_args.kwargs.get("chunk_type") == "raw" + async def test_top_k_zero_skips_retrieval(self, mock_svc, monkeypatch): + """bb-szm: streaming chat with an explicit top_k:0 bypasses retrieval and + emits an empty SOURCES event -- a pure-LLM call, not a grounded one.""" + monkeypatch.setattr(cfg, "chat_mode", ChatMode.SEARCH.value) + mock_svc.searcher.direct_messages.return_value = [{"role": "user", "content": "q"}] + monkeypatch.setattr( + _rag_h, "dispatch_chat_stream", lambda req: _canonical_text_stream(["direct"]) + ) + events = [e async for e in handlers.chat_stream("q", [], top_k=0)] + mock_svc.searcher.build_rag_context.assert_not_called() + mock_svc.searcher.direct_messages.assert_called_once() + sources_event = next(e for e in events if e and e.startswith("event: sources")) + assert json.loads(sources_event.split("data: ")[1].strip()) == [] + + async def test_reasoning_only_stream_emits_notice(self, mock_svc, monkeypatch): + """bb-cpu: a reasoning-only stream (no final answer) emits the exhaustion + notice as a token, even with show_reasoning off, so it isn't silent.""" + from lilbee.retrieval.reasoning import REASONING_EXHAUSTED_NOTICE + + monkeypatch.setattr(cfg, "show_reasoning", False) + mock_svc.searcher.build_rag_context.return_value = _rag_return() + monkeypatch.setattr( + _rag_h, + "dispatch_chat_stream", + lambda req: _canonical_text_stream(["", "endless reasoning", ""]), + ) + events = [e async for e in handlers.chat_stream("q", [])] + token_events = [e for e in events if e and e.startswith("event: token")] + assert any(REASONING_EXHAUSTED_NOTICE in e for e in token_events) + async def test_sources_event_carries_cited_subset(self, mock_svc, monkeypatch): """The chat SOURCES event emits the cited subset (what the answer referenced), matching the non-stream cited_sources contract, not the full retrieved list.""" diff --git a/tools/wheel-build/build_llama_server.sh b/tools/wheel-build/build_llama_server.sh index 15e13447d..fa1ee38bc 100755 --- a/tools/wheel-build/build_llama_server.sh +++ b/tools/wheel-build/build_llama_server.sh @@ -24,7 +24,7 @@ _DEFAULT_LLAMA_CPP_VERSION="0.3.30" # Built from source (deterministic, no release-asset-name guessing); the wheel-build # job provides the Go toolchain. Bump deliberately. _LLAMA_SWAP_VERSION="v223" -_GGUF_PARSER_REF="v0.9.3" +_GGUF_PARSER_REF="v0.24.1" backend="${BACKEND:?BACKEND is required}" build_dir="${LLAMA_BUILD_DIR:-/tmp/llama-build}" From 2f062d0caa8678554c6e3256f1ed20c1b683350b Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 4 Jul 2026 18:57:49 -0400 Subject: [PATCH 34/77] fix(chat): keep the reasoning-exhausted notice out of memory extraction Follow-up to the local-model-api fixes (#482). The reasoning-exhausted notice is a synthetic non-answer, so -- like the search-needs-embedder refusal -- it must not seed memory or count as a citation source. Skip extraction on both the one-shot and streaming chat paths. Also cover bare-repo recovery of a quant-subdir split GGUF from a raw cache. --- src/lilbee/server/handlers/rag.py | 16 +++++++++++++--- tests/test_registry.py | 19 +++++++++++++++++++ tests/test_server_handlers.py | 9 +++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/lilbee/server/handlers/rag.py b/src/lilbee/server/handlers/rag.py index 47fd8b8a5..a5fe3639a 100644 --- a/src/lilbee/server/handlers/rag.py +++ b/src/lilbee/server/handlers/rag.py @@ -344,9 +344,12 @@ async def chat( if not answer.strip() and text.strip(): # The model emitted only reasoning (stripped to nothing) and no final # answer. Surface that distinctly instead of a silent empty string the - # caller can't tell apart from a legitimate empty response (bb-cpu). + # caller can't tell apart from a legitimate empty response (bb-cpu). The + # synthetic notice is not an answer, so -- like the search-needs-embedder + # refusal -- it doesn't seed memory. answer = REASONING_EXHAUSTED_NOTICE - await _store_extracted_memories(question, answer) + else: + await _store_extracted_memories(question, answer) return AskResponse( answer=answer, sources=[CleanedChunk(**clean_result(s)) for s in sources], @@ -410,7 +413,14 @@ async def _stream_chat_response( answer_parts: list[str] = [] try: async for event in _cap_aware_chat_events(req): - if isinstance(event, StreamToken) and not event.is_reasoning: + if ( + isinstance(event, StreamToken) + and not event.is_reasoning + # The reasoning-exhausted notice is streamed to the client but is + # not a real answer: keep it out of answer_parts so it seeds no + # memory and isn't treated as a citation source (bb-cpu). + and event.content != REASONING_EXHAUSTED_NOTICE + ): answer_parts.append(event.content) yield _sse_for_chat_event(event) except Exception as exc: diff --git a/tests/test_registry.py b/tests/test_registry.py index 9b77a1bb3..2fb389750 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -451,6 +451,25 @@ def test_bare_repo_recovers_split_gguf_from_raw_cache(self, tmp_path: Path) -> N assert len(manifest.shard_blobs) == 2 assert manifest.total_size_bytes == sum(len(f"shard-{n}".encode()) for n in (1, 2, 3)) + def test_bare_repo_recovers_subdir_split_gguf_from_raw_cache(self, tmp_path: Path) -> None: + """Real quant repos (e.g. unsloth) place their shards under a quant subdir; + a bare-repo ref must recover that split set from a raw cache too, not just a + flat one.""" + registry = ModelRegistry(tmp_path) + repo = "unsloth/MiniMax-M2-GGUF" + for n in (1, 2, 3): + _seed_hf_cache( + tmp_path, + repo=repo, + filename=f"Q4_K_M/MiniMax-M2-Q4_K_M-0000{n}-of-00003.gguf", + content=f"shard-{n}".encode(), + ) + resolved = registry.resolve(repo) + assert resolved == registry.resolve(f"{repo}/Q4_K_M/MiniMax-M2-Q4_K_M-00001-of-00003.gguf") + assert resolved.parent.name == "Q4_K_M" # co-located under the quant subdir + for n in (2, 3): + assert (resolved.parent / f"MiniMax-M2-Q4_K_M-0000{n}-of-00003.gguf").exists() + def test_shard_paths_returns_every_split_shard(self, tmp_path: Path) -> None: registry = ModelRegistry(tmp_path) repo = "ggml-org/gpt-oss-120b-GGUF" diff --git a/tests/test_server_handlers.py b/tests/test_server_handlers.py index b6cab1188..b073a362e 100644 --- a/tests/test_server_handlers.py +++ b/tests/test_server_handlers.py @@ -806,8 +806,17 @@ async def test_reasoning_only_answer_surfaces_notice(self, mock_svc, monkeypatch monkeypatch.setattr(cfg, "chat_mode", ChatMode.SEARCH.value) monkeypatch.setattr(cfg, "show_reasoning", False) mock_svc.searcher.build_rag_context.return_value = _rag_return() + extract_calls: list[str] = [] + + async def _spy_extract(_question, answer): + extract_calls.append(answer) + return [] + + monkeypatch.setattr(_rag_h, "_store_extracted_memories", _spy_extract) result = await handlers.chat("q", []) assert result.answer == REASONING_EXHAUSTED_NOTICE + # The synthetic notice is not an answer: it must not seed memory. + assert extract_calls == [] class TestChatStream: From b188daa0447142e62ac5a26a23e9dfeaa17ab3ee Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sun, 5 Jul 2026 00:14:22 -0400 Subject: [PATCH 35/77] ingest: LILBEE_OCR_FORCE env lever for garbage-text-layer scans Scans with a whitespace-only or invisible text layer pass the has-text gate, skip OCR, and extract zero chunks. LILBEE_OCR_FORCE=1 sets an impossible non-whitespace floor so every page falls through to the OCR backend -- a targeted re-ingest lever for exactly those files; unset, the thresholds stay at xberg defaults and behavior is unchanged. --- src/lilbee/data/ingest/extract.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/lilbee/data/ingest/extract.py b/src/lilbee/data/ingest/extract.py index e40f76a72..b7d7c8247 100644 --- a/src/lilbee/data/ingest/extract.py +++ b/src/lilbee/data/ingest/extract.py @@ -111,12 +111,33 @@ def _ocr_config(ocr_token: str | None) -> OcrConfig: return OcrConfig(enabled=False) if config.vision_model: options = backend_options_for(ocr_token) if ocr_token else None - return OcrConfig(backend=OcrBackendName.LILBEE_VISION, backend_options=options) + return OcrConfig( + backend=OcrBackendName.LILBEE_VISION, + backend_options=options, + quality_thresholds=_forced_ocr_thresholds(), + ) # xberg requires a non-empty language list (4.x defaulted to English; # 5.x errors on an empty one). cfg.ocr_language is validated non-empty. return OcrConfig(backend=OcrBackendName.TESSERACT, language=list(config.ocr_language)) +def _forced_ocr_thresholds() -> "OcrQualityThresholds | None": + """OCR-forcing thresholds when LILBEE_OCR_FORCE=1, else None (xberg defaults). + + Some scans carry a garbage text layer (whitespace-only or invisible text + objects), so the has-text-layer gate skips OCR and extraction yields zero + chunks. An impossible non-whitespace floor makes every page fail the + quality gate and fall through to OCR -- a targeted-reingest lever, not a + default: normal runs must keep native-first extraction.""" + import os + + if os.environ.get("LILBEE_OCR_FORCE", "").strip().lower() not in {"1", "true", "yes"}: + return None + from xberg import OcrQualityThresholds + + return OcrQualityThresholds(min_total_non_whitespace=10**9) + + def extraction_config(mode: ExtractMode, *, ocr_token: str | None = None) -> ExtractionConfig: """Build ExtractionConfig for the given extraction mode.""" from xberg import ExtractionConfig, PageConfig From 577430a656290036facfb7e1e8a4409678c7d8aa Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sun, 5 Jul 2026 00:56:04 -0400 Subject: [PATCH 36/77] Manual placement derives planner-style tensor splits instead of even shares --- src/lilbee/providers/fleet/placement.py | 13 ++++- .../fleet/test_placement_from_spec.py | 48 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/lilbee/providers/fleet/placement.py b/src/lilbee/providers/fleet/placement.py index 52b704301..62c36ff2a 100644 --- a/src/lilbee/providers/fleet/placement.py +++ b/src/lilbee/providers/fleet/placement.py @@ -317,7 +317,7 @@ def placement_from_spec( instances: list[InstancePlan] = [] for role in active_roles: rp = _required_entry(spec, role, device_capacity) - ratio = rp.tensor_split or tuple(1 for _ in rp.devices) + ratio = rp.tensor_split or _derived_split(rp.devices, remaining) per_device = estimate_peak(role, ratio) split = ratio if len(rp.devices) > 1 else () for replica in range(rp.replicas): @@ -330,6 +330,17 @@ def placement_from_spec( return Placement(instances=tuple(instances), unplaceable_roles=()) +def _derived_split(devices: tuple[int, ...], remaining: dict[int, float]) -> tuple[int, ...]: + """Planner-style split for a spec entry without an explicit ``tensor_split``. + + Each card's shard tracks its remaining usable VRAM (whole GiB, min 1) — the + same proportions the auto planner computes — so a card already carrying + other roles takes a smaller share. An even split would charge every card the + same shard and falsely reject layouts the planner itself serves (bb-lt7). + """ + return tuple(max(1, int(remaining[idx] / 1024**3)) for idx in devices) + + def _required_entry( spec: PlacementSpec, role: WorkerRole, device_capacity: dict[int, int] ) -> RolePlacement: diff --git a/tests/providers/fleet/test_placement_from_spec.py b/tests/providers/fleet/test_placement_from_spec.py index 97e7231c2..71490d214 100644 --- a/tests/providers/fleet/test_placement_from_spec.py +++ b/tests/providers/fleet/test_placement_from_spec.py @@ -78,3 +78,51 @@ def test_errors_when_two_roles_overbook_one_card(): placement_from_spec( spec, (WorkerRole.CHAT, WorkerRole.EMBED), {0: 80 * GIB}, estimate_peak=est ) + + +def _proportional_peak(totals_gib): + # Like the real estimator: a role's total bytes split proportionally to ratio. + def estimate(role, ratio): + total = totals_gib[role] * GIB + s = sum(ratio) + return tuple(int(total * r / s) for r in ratio) + + return estimate + + +def test_derives_planner_style_split_when_spec_has_none(): + # bb-lt7: three 48 GiB cards with the embedder resident on card 0. An even + # chat split (~36.7 GiB per card) overflows card 0 (43.2 usable minus 8.5 + # embed = 34.7), so the even-split validator rejects the exact layout the + # auto planner serves. A remaining-proportional split shrinks card 0's + # shard and the same layout fits. + spec = PlacementSpec( + { + WorkerRole.EMBED: RolePlacement(devices=(0,)), + WorkerRole.CHAT: RolePlacement(devices=(0, 1, 2)), + } + ) + est = _proportional_peak({WorkerRole.CHAT: 110, WorkerRole.EMBED: 8.5}) + placement = placement_from_spec( + spec, + (WorkerRole.EMBED, WorkerRole.CHAT), # planning registers non-chat roles first + {0: 48 * GIB, 1: 48 * GIB, 2: 48 * GIB}, + estimate_peak=est, + ) + assert placement.unplaceable_roles == () + chat = next(i for i in placement.instances if i.role is WorkerRole.CHAT) + assert len(chat.tensor_split) == 3 + assert chat.tensor_split[0] < chat.tensor_split[1] # co-resident card takes the smaller shard + + +def test_explicit_tensor_split_still_wins(): + # A spec that names its own split is honored verbatim, even when uneven. + spec = PlacementSpec( + {WorkerRole.CHAT: RolePlacement(devices=(0, 1), tensor_split=(3, 1))} + ) + est = _proportional_peak({WorkerRole.CHAT: 40}) + placement = placement_from_spec( + spec, (WorkerRole.CHAT,), {0: 80 * GIB, 1: 80 * GIB}, estimate_peak=est + ) + chat = next(i for i in placement.instances if i.role is WorkerRole.CHAT) + assert chat.tensor_split == (3, 1) From ed3fe54a6e4df7440b7eb2c25fd9f9866fd2950f Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 4 Jul 2026 15:19:15 -0400 Subject: [PATCH 37/77] build(xberg): integrate 1.0.0rc7 (canonical kreuzberg main) Bump the local trial wheel from rc6 to 1.0.0rc7, built from canonical kreuzberg main 6fa598759f ("chore: regenreated"). Its committed bindings were regenerated with alef 0.32.1, which carries the OcrBackend host-method forwarding fix (kreuzberg-3u2) on top of the public options.OcrConfig Protocol (kreuzberg-u4r) already present in rc6. No xberg runtime change vs rc6; the delta is regenerated bindings. The forwarding fix makes VisionOcrBackend's capability methods (supports_table_detection, supports_document_processing, emits_structured_markdown, supported_languages, version) live rather than defaulted, so they are kept. Full suite green at 100% coverage; the real-xberg Tesseract integration test and the native-first / shape / verify_excerpt oracle checks all pass against rc7 with zero xberg type:ignore remaining. --- pyproject.toml | 10 +++++----- uv.lock | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 089d116f1..6ea820fc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,11 +94,11 @@ packages = ["src/lilbee"] lilbee-engine = { path = "packaging/engine-wheel" } # TODO: pre-publication trial wheel; resolve it from the local build until it # lands on PyPI, then drop this source and lock from the index. Built from xberg -# 1.0.0rc6 (kreuzberg e702310938... -> e94bc15621) with the python bindings -# regenerated off alef main (2c3803e: builtins-qualified stubs + let_unit_value, -# atop the public-OcrConfig-in-Protocol contract fix). Pure binding fixes; no -# xberg runtime change vs the committed rc6 bindings. -xberg = { path = "/Users/tobias/projects/kreuzberg/dist/xberg-1.0.0rc6-cp310-abi3-macosx_11_0_arm64.whl" } +# 1.0.0rc7 (kreuzberg main 6fa598759f "chore: regenreated"), whose committed +# bindings were regenerated with alef 0.32.1 -- carries the OcrBackend host-method +# forwarding fix (kreuzberg-3u2), the public options.OcrConfig Protocol +# (kreuzberg-u4r), and builtins-qualified stubs. +xberg = { path = "/Users/tobias/projects/kreuzberg/dist/xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl" } [dependency-groups] dev = [ diff --git a/uv.lock b/uv.lock index 6cdaf8191..a6c66f7b7 100644 --- a/uv.lock +++ b/uv.lock @@ -1709,7 +1709,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "typing-extensions", specifier = ">=4.5.0" }, { name = "uvicorn", specifier = ">=0.30" }, - { name = "xberg", path = "../../../../kreuzberg/dist/xberg-1.0.0rc6-cp310-abi3-macosx_11_0_arm64.whl" }, + { name = "xberg", path = "../../../../kreuzberg/dist/xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl" }, ] provides-extras = ["litellm", "graph", "crawler", "cuda12", "release"] @@ -4450,10 +4450,10 @@ wheels = [ [[package]] name = "xberg" -version = "1.0.0rc6" -source = { path = "../../../../kreuzberg/dist/xberg-1.0.0rc6-cp310-abi3-macosx_11_0_arm64.whl" } +version = "1.0.0rc7" +source = { path = "../../../../kreuzberg/dist/xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ed3a85dc002b894b47049554cf3641c409c7d448c55fda0be25364bfcdba0fbe" }, + { filename = "xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:aa94891bfb7e039817aa078a5a535304d65dacc2c1e6fbe1a7f5c346967fcedf" }, ] [[package]] From 3e90f613db84eed73b740cda475cad36b9c8f4ac Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sun, 5 Jul 2026 01:57:17 -0400 Subject: [PATCH 38/77] build(xberg): refresh 1.0.0rc7 wheel with tokenizer-backend registry Swap the local trial wheel for a newer 1.0.0rc7 build off kreuzberg main that adds the pluggable tokenizer-backend registry (kreuzberg-1h6): register_tokenizer_backend / unregister_tokenizer_backend / list_tokenizer_backends / clear_tokenizer_backends plus the TokenizerBackend Protocol. The change is purely additive over the prior rc7 -- every symbol lilbee imports is unchanged and lilbee does not use the tokenizer API. exceptions.py and options.py differ only by the regenerated alef header hash; api.py gains the four functions and is otherwise reformatted. Full suite green at 100% coverage; the real-xberg Tesseract integration test and the native-first / shape / verify_excerpt oracle checks all pass against the new build with zero xberg type:ignore remaining. --- pyproject.toml | 9 +++++---- uv.lock | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6ea820fc8..71d686602 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,10 +94,11 @@ packages = ["src/lilbee"] lilbee-engine = { path = "packaging/engine-wheel" } # TODO: pre-publication trial wheel; resolve it from the local build until it # lands on PyPI, then drop this source and lock from the index. Built from xberg -# 1.0.0rc7 (kreuzberg main 6fa598759f "chore: regenreated"), whose committed -# bindings were regenerated with alef 0.32.1 -- carries the OcrBackend host-method -# forwarding fix (kreuzberg-3u2), the public options.OcrConfig Protocol -# (kreuzberg-u4r), and builtins-qualified stubs. +# 1.0.0rc7 built from kreuzberg main, whose committed bindings were +# regenerated with alef -- carries the OcrBackend host-method forwarding fix +# (kreuzberg-3u2), the public options.OcrConfig Protocol (kreuzberg-u4r), +# builtins-qualified stubs, and the pluggable tokenizer-backend registry +# (kreuzberg-1h6). The tokenizer API is purely additive; lilbee does not use it. xberg = { path = "/Users/tobias/projects/kreuzberg/dist/xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl" } [dependency-groups] diff --git a/uv.lock b/uv.lock index a6c66f7b7..115b68ef7 100644 --- a/uv.lock +++ b/uv.lock @@ -4453,7 +4453,7 @@ name = "xberg" version = "1.0.0rc7" source = { path = "../../../../kreuzberg/dist/xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:aa94891bfb7e039817aa078a5a535304d65dacc2c1e6fbe1a7f5c346967fcedf" }, + { filename = "xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7daed6ea563de914a4aa6d9098213521f3020a937d8e0903f5a46e5bbb87ee05" }, ] [[package]] From 0e5feff69d77ae6a330edaf2ecb2c469cf73805b Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:37:31 -0400 Subject: [PATCH 39/77] build(xberg): integrate 1.0.0rc9 with tokenizer-backed chunk sizing Bump the local trial wheel to 1.0.0rc9, built from kreuzberg main (293f1eaadc, bindings regenerated with alef 0.32.6). rc9 carries the merged tokenizer-backend sizing feature (PR #1196 / kreuzberg-1h6): ChunkSizing::Tokenizer resolves a registered TokenizerBackend name registry-first, so a caller can size chunks with its own embedder tokenizer instead of a HuggingFace proxy or a char heuristic. Also picks up the OcrBackend forwarding fix (kreuzberg-3u2) and the public options.OcrConfig Protocol (kreuzberg-u4r) already relied on. --- pyproject.toml | 13 +++++++------ uv.lock | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 71d686602..58dc17445 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,12 +94,13 @@ packages = ["src/lilbee"] lilbee-engine = { path = "packaging/engine-wheel" } # TODO: pre-publication trial wheel; resolve it from the local build until it # lands on PyPI, then drop this source and lock from the index. Built from xberg -# 1.0.0rc7 built from kreuzberg main, whose committed bindings were -# regenerated with alef -- carries the OcrBackend host-method forwarding fix -# (kreuzberg-3u2), the public options.OcrConfig Protocol (kreuzberg-u4r), -# builtins-qualified stubs, and the pluggable tokenizer-backend registry -# (kreuzberg-1h6). The tokenizer API is purely additive; lilbee does not use it. -xberg = { path = "/Users/tobias/projects/kreuzberg/dist/xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl" } +# 1.0.0rc9 built from kreuzberg main (293f1eaadc), bindings regenerated with +# alef 0.32.6. Carries the OcrBackend host-method forwarding fix (kreuzberg-3u2), +# the public options.OcrConfig Protocol (kreuzberg-u4r), and the merged +# tokenizer-backend sizing feature (PR #1196 / kreuzberg-1h6): ChunkSizing +# resolves a registered TokenizerBackend name registry-first, so lilbee can size +# chunks with its own embedder tokenizer instead of the char-budget heuristic. +xberg = { path = "/Users/tobias/projects/kreuzberg/dist/xberg-1.0.0rc9-cp310-abi3-macosx_11_0_arm64.whl" } [dependency-groups] dev = [ diff --git a/uv.lock b/uv.lock index 115b68ef7..dd767afdf 100644 --- a/uv.lock +++ b/uv.lock @@ -1709,7 +1709,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "typing-extensions", specifier = ">=4.5.0" }, { name = "uvicorn", specifier = ">=0.30" }, - { name = "xberg", path = "../../../../kreuzberg/dist/xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl" }, + { name = "xberg", path = "../../../../kreuzberg/dist/xberg-1.0.0rc9-cp310-abi3-macosx_11_0_arm64.whl" }, ] provides-extras = ["litellm", "graph", "crawler", "cuda12", "release"] @@ -4450,10 +4450,10 @@ wheels = [ [[package]] name = "xberg" -version = "1.0.0rc7" -source = { path = "../../../../kreuzberg/dist/xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl" } +version = "1.0.0rc9" +source = { path = "../../../../kreuzberg/dist/xberg-1.0.0rc9-cp310-abi3-macosx_11_0_arm64.whl" } wheels = [ - { filename = "xberg-1.0.0rc7-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7daed6ea563de914a4aa6d9098213521f3020a937d8e0903f5a46e5bbb87ee05" }, + { filename = "xberg-1.0.0rc9-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:86b4248c863f69a9fe575f49ceaddeb395882115c11ef1598cd2ed49748f9a7c" }, ] [[package]] From c825bc358d201d836ef9dd09f97012845be9f523 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:37:45 -0400 Subject: [PATCH 40/77] feat(chunk): size chunks with the embedder's own tokenizer Add an opt-in `token_sizing` setting that budgets chunks in real embedder tokens instead of the chars-per-token heuristic, which mis-sizes token-dense text (dates, IDs, code) by 2-4x. When it is on, the plain and heading chunkers pass chunk_size/chunk_overlap straight through as a token budget and attach xberg's ChunkSizing pointed at a registered tokenizer backend; when off, the character budget is unchanged. The backend (LilbeeTokenizerBackend) routes token counts to the embedder's own /tokenize via provider.count_tokens, so the budget agrees with the model that later embeds the chunk. xberg calls it synchronously inside the chunk sizer and requires a non-zero count for non-empty text, so any failure (embedder not loaded, a cloud provider with no local tokenizer) degrades to a conservative character estimate rather than raising. It is registered only while token_sizing is on, and re-synced whenever the setting is toggled or the provider is rebuilt. count_tokens is added to the provider protocol and implemented on the fleet provider (exact, via /tokenize), routed by the routing provider, and raised as NotImplementedError by the SDK provider (no local tokenizer). token_sizing is reindex-scoped: changing it re-partitions chunks. The semantic chunker sizes by characters and ignores ChunkSizing, so it stays on the character budget regardless of this setting. --- src/lilbee/app/services.py | 54 ++++++++++++--- src/lilbee/app/settings.py | 9 +++ src/lilbee/app/settings_map.py | 6 ++ src/lilbee/core/config/model.py | 6 ++ src/lilbee/data/chunk.py | 39 +++++++++-- src/lilbee/data/ingest/types.py | 9 +++ src/lilbee/data/tokenizer_backend.py | 67 +++++++++++++++++++ src/lilbee/providers/base.py | 9 +++ src/lilbee/providers/fleet/client.py | 4 ++ src/lilbee/providers/fleet/provider.py | 11 ++++ src/lilbee/providers/routing_provider.py | 4 ++ src/lilbee/providers/sdk_llm_provider.py | 5 ++ tests/test_chunker.py | 83 ++++++++++++++++++++++++ tests/test_fleet_client.py | 11 ++++ tests/test_fleet_provider.py | 16 +++++ tests/test_providers.py | 33 ++++++++++ tests/test_services.py | 76 ++++++++++++++++++++++ tests/test_tokenizer_backend.py | 56 ++++++++++++++++ 18 files changed, 482 insertions(+), 16 deletions(-) create mode 100644 src/lilbee/data/tokenizer_backend.py create mode 100644 tests/test_tokenizer_backend.py diff --git a/src/lilbee/app/services.py b/src/lilbee/app/services.py index b7a109716..ef8bcb636 100644 --- a/src/lilbee/app/services.py +++ b/src/lilbee/app/services.py @@ -158,12 +158,13 @@ def build_services( reconciliation is a global-cfg concern owned by :func:`get_services`, not done here. - Side effect: registers *provider* as xberg's process-global OCR and embedding - backend (:func:`sync_vision_ocr_backend` / :func:`sync_embedding_backend`) so - scanned-page OCR and semantic-chunk boundary detection route through it. xberg's - registry is a single global slot, so the most recently built container wins; - this is why registration lives here (every container, singleton or per-instance - library, binds its own provider) rather than only in :func:`get_services`. + Side effect: registers *provider* as xberg's process-global OCR, embedding, and + tokenizer backend (:func:`sync_vision_ocr_backend` / :func:`sync_embedding_backend` + / :func:`sync_tokenizer_backend`) so scanned-page OCR, semantic-chunk boundary + detection, and token-budgeted chunk sizing route through it. xberg's registry is a + single global slot, so the most recently built container wins; this is why + registration lives here (every container, singleton or per-instance library, binds + its own provider) rather than only in :func:`get_services`. """ from lilbee.catalog.hf_client import HfClient from lilbee.data.store import Store @@ -179,12 +180,14 @@ def build_services( from lilbee.runtime.ingest_lock import IngestLockRegistry provider = provider or create_provider(config) - # Register this provider as xberg's OCR + embedding backend so scanned-page OCR - # and semantic-chunk boundary detection route through it. Bound here rather than - # in get_services so the library API's per-instance containers register too; both - # backends read the live cfg and re-bind whenever the provider is rebuilt. + # Register this provider as xberg's OCR + embedding + tokenizer backend so + # scanned-page OCR, semantic-chunk boundary detection, and token-budgeted chunk + # sizing route through it. Bound here rather than in get_services so the library + # API's per-instance containers register too; each backend reads the live cfg and + # re-binds whenever the provider is rebuilt. sync_vision_ocr_backend(provider) sync_embedding_backend(provider) + sync_tokenizer_backend(provider) registry = registry or ModelRegistry(config.models_dir) store = Store(config) embedder = Embedder(config, provider) @@ -323,6 +326,37 @@ def sync_embedding_backend(provider: LLMProvider) -> None: ) +def sync_tokenizer_backend(provider: LLMProvider) -> None: + """Register lilbee's embedder tokenizer as xberg's chunk-sizing backend. + + Driven by ``cfg.token_sizing``: registered while it is on, removed when off. + Gated rather than always-on because xberg validates a tokenizer backend at + registration by probing ``count_tokens``, so registering only when the feature + is on keeps that probe (and its embedder round-trip) off the default path. The + backend captures ``provider.count_tokens``, so it must re-bind whenever the + provider is rebuilt (``reset_services``) -- otherwise xberg's registry keeps + counting with the shut-down provider. + """ + from xberg import ( + list_tokenizer_backends, + register_tokenizer_backend, + unregister_tokenizer_backend, + ) + + from lilbee.core.config import cfg + from lilbee.data.ingest.types import TokenizerBackendName + from lilbee.data.tokenizer_backend import LilbeeTokenizerBackend + + registered = TokenizerBackendName.LILBEE in list_tokenizer_backends() + if cfg.token_sizing: + # Re-register so the backend always binds to the current provider. + if registered: + unregister_tokenizer_backend(TokenizerBackendName.LILBEE) + register_tokenizer_backend(LilbeeTokenizerBackend(count_fn=provider.count_tokens)) + elif registered: + unregister_tokenizer_backend(TokenizerBackendName.LILBEE) + + def set_services(services: Services | None) -> None: """Replace the cached Services singleton (for testing).""" _state.singleton = services diff --git a/src/lilbee/app/settings.py b/src/lilbee/app/settings.py index e92fe4315..cd1c877a5 100644 --- a/src/lilbee/app/settings.py +++ b/src/lilbee/app/settings.py @@ -319,6 +319,15 @@ def _invalidate_caches(changed_keys: set[str]) -> None: if changed_keys & LOAD_AFFECTING_KEYS: # heavy: app.services pulls the provider stack + lancedb (~70 ms) _reload_changed_roles(changed_keys) + if "token_sizing" in changed_keys: + # Register/unregister lilbee's xberg tokenizer backend when token_sizing is + # toggled (via any settings path), so chunk sizing picks up the change + # without waiting for a services rebuild. + from lilbee.app.services import peek_services, sync_tokenizer_backend + + services = peek_services() + if services is not None: + sync_tokenizer_backend(services.provider) if changed_keys & PROVIDER_API_KEYS: # heavy: sdk_llm_provider pulls litellm fanout (~145 ms) from lilbee.providers.sdk_llm_provider import inject_provider_keys diff --git a/src/lilbee/app/settings_map.py b/src/lilbee/app/settings_map.py index 44bc63b70..ba4e6a4aa 100644 --- a/src/lilbee/app/settings_map.py +++ b/src/lilbee/app/settings_map.py @@ -152,6 +152,12 @@ def get_default(key: str) -> object: group=SettingGroup.INGEST, help_text="Topic-boundary similarity threshold, 0.0-1.0, used when semantic chunking is on", ), + "token_sizing": SettingDef( + bool, + nullable=False, + group=SettingGroup.INGEST, + help_text="Size chunks by real embedder tokens, not chars (changes invalidate the index)", + ), "embedding_model": SettingDef( str, nullable=False, diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index fea916c49..4b2f1bfc5 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -139,6 +139,12 @@ class Config(BaseSettings): ocr_language: list[str] = ConfigField(default_factory=lambda: ["eng"], writable=True) semantic_chunking: bool = ConfigField(default=False, writable=True) topic_threshold: float = ConfigField(default=0.75, ge=0.0, le=1.0, writable=True) + # Size chunk budgets (chunk_size/chunk_overlap) in real tokens from the + # embedder's own tokenizer via xberg's registered tokenizer backend, instead of + # the chars-per-token heuristic. Off by default: turning it on re-partitions + # chunks, so a library must be reindexed. Applies to the plain and heading + # chunkers; the semantic chunker sizes by characters and ignores it. + token_sizing: bool = ConfigField(default=False, writable=True, reindex=True) server_host: str = "127.0.0.1" server_port: int = Field(default=0, ge=0, le=65535) cors_origins: list[str] = Field(default_factory=list) diff --git a/src/lilbee/data/chunk.py b/src/lilbee/data/chunk.py index d2019c486..b1cfccf3e 100644 --- a/src/lilbee/data/chunk.py +++ b/src/lilbee/data/chunk.py @@ -7,7 +7,7 @@ from lilbee.core.config import active_config if TYPE_CHECKING: - from xberg import ChunkingConfig, EmbeddingConfig + from xberg import ChunkingConfig, ChunkSizing, EmbeddingConfig CHARS_PER_TOKEN = 4 @@ -23,6 +23,30 @@ def _char_budget() -> tuple[int, int]: return max_chars, max_overlap +def _size_params() -> tuple[int, int, ChunkSizing | None]: + """Return (max, overlap, sizing) for the plain and heading chunkers. + + With ``cfg.token_sizing`` on, the budget is a raw token count and ``sizing`` + routes to lilbee's registered tokenizer backend, so ``chunk_size`` is a real + token ceiling. Otherwise the character heuristic with no sizing (xberg's default + character sizer). The semantic chunker does not use this -- it sizes by + characters and ignores ChunkSizing.""" + config = active_config() + if config.token_sizing: + from xberg import ChunkSizing + + from lilbee.data.ingest.types import TokenizerBackendName + + overlap = min(config.chunk_overlap, config.chunk_size // 2) + return ( + config.chunk_size, + overlap, + ChunkSizing(type="tokenizer", model=TokenizerBackendName.LILBEE), + ) + max_chars, max_overlap = _char_budget() + return max_chars, max_overlap, None + + def _semantic_embedding_config() -> EmbeddingConfig: """EmbeddingConfig for semantic chunking. Boundary-detection embeddings route to lilbee's embedder, registered as xberg's plugin backend in @@ -41,10 +65,11 @@ def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: """Build an xberg ChunkingConfig from the current cfg.""" from xberg import ChunkingConfig - max_chars, max_overlap = _char_budget() - config = active_config() if use_semantic and config.semantic_chunking: + # The semantic chunker sizes by characters and ignores ChunkSizing, so it + # stays on the character budget regardless of cfg.token_sizing. + max_chars, max_overlap = _char_budget() return ChunkingConfig( chunker_type=_SEMANTIC_CHUNKER, embedding=_semantic_embedding_config(), @@ -52,7 +77,8 @@ def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: max_characters=max_chars, overlap=max_overlap, ) - return ChunkingConfig(max_characters=max_chars, overlap=max_overlap) + max_size, max_overlap, sizing = _size_params() + return ChunkingConfig(max_characters=max_size, overlap=max_overlap, sizing=sizing) def chunk_text( @@ -71,10 +97,11 @@ def chunk_text( from lilbee.data.xberg_extract import extract_document if heading_context: - max_chars, max_overlap = _char_budget() + max_size, max_overlap, sizing = _size_params() chunking = ChunkingConfig( - max_characters=max_chars, + max_characters=max_size, overlap=max_overlap, + sizing=sizing, chunker_type=_MARKDOWN_CHUNKER, prepend_heading_context=True, ) diff --git a/src/lilbee/data/ingest/types.py b/src/lilbee/data/ingest/types.py index babdf36f1..33878748e 100644 --- a/src/lilbee/data/ingest/types.py +++ b/src/lilbee/data/ingest/types.py @@ -62,6 +62,15 @@ class EmbeddingBackendName(StrEnum): LILBEE = "lilbee" +class TokenizerBackendName(StrEnum): + """Tokenizer backends registered with xberg. lilbee registers its embedder's + tokenizer so ChunkSizing counts chunk budgets in the same tokens the embedder + consumes, instead of a chars-per-token heuristic. Separate registry from the + embedding backend, so sharing the ``lilbee`` name is fine.""" + + LILBEE = "lilbee" + + class ExtractMode(StrEnum): """Extraction topology: paginated (PDFs/images) vs markdown output (text formats).""" diff --git a/src/lilbee/data/tokenizer_backend.py b/src/lilbee/data/tokenizer_backend.py new file mode 100644 index 000000000..9694eb5ba --- /dev/null +++ b/src/lilbee/data/tokenizer_backend.py @@ -0,0 +1,67 @@ +"""lilbee's embedder tokenizer exposed as a xberg plugin tokenizer backend. + +xberg's chunk sizer can budget chunks in tokens, but only with tokenizers it loads +itself (a HuggingFace id or a local ``tokenizer.json``). lilbee's embedder is a +GGUF model served by llama-server, whose vocab is not published that way. +Registering this backend lets ChunkSizing count chunk budgets with the exact +tokenizer the embedder consumes, so ``chunk_size`` is a real token ceiling instead +of a chars-per-token guess (which mis-sizes token-dense text by 2-4x). +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from lilbee.data.ingest.types import TokenizerBackendName + +if TYPE_CHECKING: + from collections.abc import Callable + +log = logging.getLogger(__name__) + +# Conservative (over-counting) chars-per-token used only when the exact count is +# unavailable. Over-counting keeps the fallback on the safe side of the budget -- +# it splits a touch early rather than emitting an over-length chunk. Mirrors the +# embed client's own estimate (providers.fleet.client._EMBED_EST_CHARS_PER_TOKEN). +_FALLBACK_CHARS_PER_TOKEN = 3 + + +def _estimate_tokens(text: str) -> int: + """Conservative token estimate from character length (ceiling division).""" + return max(1, -(-len(text) // _FALLBACK_CHARS_PER_TOKEN)) + + +class LilbeeTokenizerBackend: + """Counts chunk-sizing tokens with lilbee's embedder tokenizer. + + ``count_fn`` is the provider's exact token count (read live, so an + embedding-model swap needs no re-registration). xberg calls ``count_tokens`` + synchronously inside the chunk sizer's boundary search -- many times per chunk, + on its own worker threads -- and requires a non-zero count for non-empty text + both at registration (a probe) and at runtime. So any failure of the exact + count degrades to the character estimate instead of raising: the embedder may be + unloaded when this runs (the registration probe, or chunking outside an ingest), + and a raise would fail registration or abort extraction while a zero would make + every span look like it fits the budget. + """ + + def __init__(self, *, count_fn: Callable[[str], int]) -> None: + self._count_fn = count_fn + + def name(self) -> str: + return TokenizerBackendName.LILBEE + + def initialize(self) -> None: ... + + def shutdown(self) -> None: ... + + def count_tokens(self, text: str) -> int: + if not text: + return 0 + try: + count = self._count_fn(text) + except Exception: # embedder unreachable/erroring: degrade, never crash chunking + log.debug("exact token count failed; using character estimate", exc_info=True) + return _estimate_tokens(text) + return count if count > 0 else _estimate_tokens(text) diff --git a/src/lilbee/providers/base.py b/src/lilbee/providers/base.py index 0fd795631..d5043616d 100644 --- a/src/lilbee/providers/base.py +++ b/src/lilbee/providers/base.py @@ -232,6 +232,15 @@ def embed(self, texts: list[str]) -> list[list[float]]: """Embed a batch of texts, return list of vectors.""" ... + def count_tokens(self, text: str) -> int: + """Exact token count of *text* under the embedding model's tokenizer. + + Raise ``NotImplementedError`` when the backend has no local tokenizer (cloud + SDK backends); token-budgeted chunk sizing then falls back to a character + estimate. + """ + ... + @overload def chat( self, diff --git a/src/lilbee/providers/fleet/client.py b/src/lilbee/providers/fleet/client.py index edb26cb95..4499956e1 100644 --- a/src/lilbee/providers/fleet/client.py +++ b/src/lilbee/providers/fleet/client.py @@ -900,6 +900,10 @@ def _detokenize(self, tokens: list[int]) -> str: _raise_for_status(resp) return str(resp.json()["content"]) + def count_tokens(self, text: str) -> int: + """Exact token count from this server's tokenizer (embed/rerank servers).""" + return len(self._tokenize(text)) + def close(self) -> None: """Close the underlying client if this instance created it.""" if self._owns_http: diff --git a/src/lilbee/providers/fleet/provider.py b/src/lilbee/providers/fleet/provider.py index 5a17c192f..109b4e0a0 100644 --- a/src/lilbee/providers/fleet/provider.py +++ b/src/lilbee/providers/fleet/provider.py @@ -744,6 +744,17 @@ def embed(self, texts: list[str]) -> list[list[float]]: clients = self._require_clients(WorkerRole.EMBED) return _call_with_failover(clients, lambda client: client.embed(texts)) + def count_tokens(self, text: str) -> int: + """Exact token count of *text* under the embedding model's tokenizer. + + Routes to the embed server's ``/tokenize`` so chunk sizing counts the same + tokens the embedder will consume. Raises ``ProviderError`` when no embed + server is configured; callers on the chunk-sizing path degrade to an + estimate rather than propagate it. + """ + clients = self._require_clients(WorkerRole.EMBED) + return _call_with_failover(clients, lambda client: client.count_tokens(text)) + def vision_ocr( self, png_bytes: bytes, model: str, prompt: str = "", *, timeout: float | None = None ) -> str: diff --git a/src/lilbee/providers/routing_provider.py b/src/lilbee/providers/routing_provider.py index 254c41eb5..2d81460ac 100644 --- a/src/lilbee/providers/routing_provider.py +++ b/src/lilbee/providers/routing_provider.py @@ -82,6 +82,10 @@ def embed(self, texts: list[str]) -> list[list[float]]: ref = parse_model_ref(cfg.embedding_model) return self._pick_backend(ref).embed(texts) + def count_tokens(self, text: str) -> int: + ref = parse_model_ref(cfg.embedding_model) + return self._pick_backend(ref).count_tokens(text) + @overload def chat( self, diff --git a/src/lilbee/providers/sdk_llm_provider.py b/src/lilbee/providers/sdk_llm_provider.py index 3ad01f719..fa3235c27 100644 --- a/src/lilbee/providers/sdk_llm_provider.py +++ b/src/lilbee/providers/sdk_llm_provider.py @@ -123,6 +123,11 @@ def embed(self, texts: list[str]) -> list[list[float]]: ) from exc return result.vectors + def count_tokens(self, text: str) -> int: + """Cloud SDK backends expose no local tokenizer, so chunk sizing falls back + to a character estimate (see LilbeeTokenizerBackend).""" + raise NotImplementedError("SDK backends have no local tokenizer for chunk sizing") + @overload def chat( self, diff --git a/tests/test_chunker.py b/tests/test_chunker.py index f2b43374f..fa0a06dd4 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -7,6 +7,7 @@ import tempfile from dataclasses import dataclass, field from pathlib import Path +from types import SimpleNamespace import pytest @@ -166,6 +167,88 @@ def test_heading_path_shares_char_budget(self, monkeypatch): assert max_overlap == 40 * CHARS_PER_TOKEN +_TOKENIZER_SIZING = '{"type":"tokenizer","model":"lilbee"}' + + +class TestTokenSizing: + """cfg.token_sizing routes the plain and heading chunkers through lilbee's + registered tokenizer backend, so chunk_size is a real token budget.""" + + def test_size_params_off_is_char_budget_without_sizing(self, monkeypatch): + from lilbee.core.config import cfg + from lilbee.data.chunk import CHARS_PER_TOKEN, _size_params + + monkeypatch.setattr(cfg, "token_sizing", False) + monkeypatch.setattr(cfg, "chunk_size", 512) + monkeypatch.setattr(cfg, "chunk_overlap", 100) + max_size, overlap, sizing = _size_params() + assert (max_size, overlap) == (512 * CHARS_PER_TOKEN, 100 * CHARS_PER_TOKEN) + assert sizing is None + + def test_size_params_on_is_raw_token_budget_with_sizing(self, monkeypatch): + from lilbee.core.config import cfg + from lilbee.data.chunk import _size_params + + monkeypatch.setattr(cfg, "token_sizing", True) + monkeypatch.setattr(cfg, "chunk_size", 512) + monkeypatch.setattr(cfg, "chunk_overlap", 100) + max_size, overlap, sizing = _size_params() + assert (max_size, overlap) == (512, 100) + assert str(sizing) == _TOKENIZER_SIZING + + def test_overlap_capped_at_half_the_token_budget(self, monkeypatch): + from lilbee.core.config import cfg + from lilbee.data.chunk import _size_params + + monkeypatch.setattr(cfg, "token_sizing", True) + monkeypatch.setattr(cfg, "chunk_size", 100) + monkeypatch.setattr(cfg, "chunk_overlap", 90) + _max_size, overlap, _sizing = _size_params() + assert overlap == 50 + + def test_default_chunker_attaches_tokenizer_sizing(self, monkeypatch): + from lilbee.core.config import cfg + from lilbee.data.chunk import build_chunking_config + + monkeypatch.setattr(cfg, "semantic_chunking", False) + monkeypatch.setattr(cfg, "token_sizing", True) + monkeypatch.setattr(cfg, "chunk_size", 512) + result = build_chunking_config() + assert result.max_characters == 512 + assert str(result.sizing) == _TOKENIZER_SIZING + + def test_semantic_chunker_ignores_token_sizing(self, monkeypatch): + """The semantic chunker sizes by characters, so it keeps the char budget even + with token_sizing on (xberg's ChunkSizing has no effect on that path).""" + from lilbee.core.config import cfg + from lilbee.data.chunk import CHARS_PER_TOKEN, build_chunking_config + + monkeypatch.setattr(cfg, "semantic_chunking", True) + monkeypatch.setattr(cfg, "token_sizing", True) + monkeypatch.setattr(cfg, "chunk_size", 512) + result = build_chunking_config() + assert result.chunker_type == "semantic" + assert result.max_characters == 512 * CHARS_PER_TOKEN + + def test_heading_path_attaches_tokenizer_sizing(self, monkeypatch): + from lilbee.core.config import cfg + from lilbee.data.chunk import chunk_text + + monkeypatch.setattr(cfg, "token_sizing", True) + captured = {} + + def fake_extract_document(data, mime_type, *, config): + chunking = config["chunking"] + captured["sizing"] = str(chunking.sizing) + captured["chunker_type"] = chunking.chunker_type + return SimpleNamespace(chunks=[]) + + monkeypatch.setattr("lilbee.data.xberg_extract.extract_document", fake_extract_document) + chunk_text("# Heading\n\nbody text", mime_type="text/markdown", heading_context=True) + assert captured["sizing"] == _TOKENIZER_SIZING + assert captured["chunker_type"] == "markdown" + + class TestMarkdownChunking: def test_splits_on_headings(self): md = ( diff --git a/tests/test_fleet_client.py b/tests/test_fleet_client.py index ee6013d6d..d722a8ddd 100644 --- a/tests/test_fleet_client.py +++ b/tests/test_fleet_client.py @@ -54,6 +54,17 @@ def _client(handler=_handler) -> LlamaServerClient: return client +def test_count_tokens_returns_server_token_count() -> None: + """count_tokens posts to the server's /tokenize and reports the token count.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/upstream/test-model/tokenize": + return httpx.Response(200, json={"tokens": [1, 2, 3, 4, 5]}) + return httpx.Response(404) + + assert _client(handler).count_tokens("hello world") == 5 + + def test_rerank_scores_pairs_via_rank_pooling() -> None: seen: dict[str, list] = {} diff --git a/tests/test_fleet_provider.py b/tests/test_fleet_provider.py index dc51a785c..ac02b9a79 100644 --- a/tests/test_fleet_provider.py +++ b/tests/test_fleet_provider.py @@ -163,6 +163,22 @@ def test_embed_routes_to_server_when_present() -> None: assert p.embed(["a"]) == [[0.1]] +def test_count_tokens_routes_to_embed_server() -> None: + client = _fake_client() + client.count_tokens.return_value = 9 + p = _provider_with_clients({WorkerRole.EMBED: [client]}) + assert p.count_tokens("hello") == 9 + client.count_tokens.assert_called_once_with("hello") + + +def test_count_tokens_without_server_raises() -> None: + from lilbee.providers.base import ProviderError + + p = _provider_with_clients({}) + with pytest.raises(ProviderError): + p.count_tokens("hello") + + def test_embed_routes_to_least_busy_replica() -> None: # Data-parallel replicas: a request goes to the idlest replica in the pool. busy, idle = _fake_client(5), _fake_client(1) diff --git a/tests/test_providers.py b/tests/test_providers.py index 7c4b064b7..52607c392 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -422,6 +422,28 @@ def test_routes_embed_to_local_engine_for_local_ref(self) -> None: result = rp.embed(["test"]) assert result == [[0.3, 0.4]] + def test_routes_count_tokens_to_local_engine_for_local_ref(self) -> None: + rp = self._make_provider() + mock_llama = mock.MagicMock() + mock_llama.count_tokens.return_value = 11 + rp._local = mock_llama + + cfg.embedding_model = ( + "nomic-ai/nomic-embed-text-v1.5-GGUF/nomic-embed-text-v1.5.Q4_K_M.gguf" + ) + assert rp.count_tokens("some text") == 11 + mock_llama.count_tokens.assert_called_once_with("some text") + + def test_routes_count_tokens_to_sdk_for_remote_ref(self) -> None: + rp = self._make_provider() + mock_sdk = mock.MagicMock() + mock_sdk.count_tokens.side_effect = NotImplementedError + rp._sdk_provider = mock_sdk + + cfg.embedding_model = "ollama/nomic-embed-text:latest" + with pytest.raises(NotImplementedError): + rp.count_tokens("some text") + def test_local_ref_never_falls_through_to_litellm(self) -> None: """Local HF refs stay on the local engine even when litellm is installed. @@ -3074,6 +3096,17 @@ def test_v1_models_sends_auth_header(self) -> None: assert headers.get("Authorization") == "Bearer sk-secret" +def test_sdk_provider_count_tokens_not_implemented() -> None: + """Cloud SDK backends have no local tokenizer, so count_tokens raises and chunk + sizing degrades to the character estimate.""" + from lilbee.providers.litellm_sdk import LitellmSdkBackend + from lilbee.providers.sdk_llm_provider import SdkLLMProvider + + provider = SdkLLMProvider(LitellmSdkBackend()) + with pytest.raises(NotImplementedError): + provider.count_tokens("hello") + + class TestSdkLLMProviderVisionOcr: """``SdkLLMProvider.vision_ocr`` translates to a multipart chat call.""" diff --git a/tests/test_services.py b/tests/test_services.py index fe5a50216..e1446341a 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -130,6 +130,82 @@ def test_registered_backend_routes_to_provider_embed(self, monkeypatch): assert backend.shutdown() is None +class TestSyncTokenizerBackend: + def _patch_xberg(self, monkeypatch, *, listed): + reg = MagicMock() + unreg = MagicMock() + monkeypatch.setattr("xberg.list_tokenizer_backends", lambda: listed) + monkeypatch.setattr("xberg.register_tokenizer_backend", reg) + monkeypatch.setattr("xberg.unregister_tokenizer_backend", unreg) + return reg, unreg + + def test_registers_when_enabled_and_absent(self, monkeypatch): + from lilbee.app.services import sync_tokenizer_backend + + monkeypatch.setattr(cfg, "token_sizing", True) + reg, unreg = self._patch_xberg(monkeypatch, listed=[]) + sync_tokenizer_backend(MagicMock()) + reg.assert_called_once() + unreg.assert_not_called() + + def test_rebinds_to_current_provider_when_already_registered(self, monkeypatch): + """A rebuilt provider must replace the stale binding: unregister then re-register, + else xberg keeps counting through the shut-down provider after reset_services.""" + from lilbee.app.services import sync_tokenizer_backend + + monkeypatch.setattr(cfg, "token_sizing", True) + reg, unreg = self._patch_xberg(monkeypatch, listed=["lilbee"]) + sync_tokenizer_backend(MagicMock()) + unreg.assert_called_once_with("lilbee") + reg.assert_called_once() + + def test_unregisters_when_disabled(self, monkeypatch): + from lilbee.app.services import sync_tokenizer_backend + + monkeypatch.setattr(cfg, "token_sizing", False) + reg, unreg = self._patch_xberg(monkeypatch, listed=["lilbee"]) + sync_tokenizer_backend(MagicMock()) + unreg.assert_called_once_with("lilbee") + reg.assert_not_called() + + def test_noop_when_disabled_and_absent(self, monkeypatch): + from lilbee.app.services import sync_tokenizer_backend + + monkeypatch.setattr(cfg, "token_sizing", False) + reg, unreg = self._patch_xberg(monkeypatch, listed=[]) + sync_tokenizer_backend(MagicMock()) + reg.assert_not_called() + unreg.assert_not_called() + + def test_registered_backend_routes_to_provider_count_tokens(self, monkeypatch): + from lilbee.app.services import sync_tokenizer_backend + + monkeypatch.setattr(cfg, "token_sizing", True) + reg, _unreg = self._patch_xberg(monkeypatch, listed=[]) + provider = MagicMock() + provider.count_tokens.return_value = 42 + sync_tokenizer_backend(provider) + backend = reg.call_args.args[0] + assert backend.name() == "lilbee" + assert backend.count_tokens("hello") == 42 + provider.count_tokens.assert_called_once_with("hello") + + def test_settings_change_syncs_tokenizer_backend(self, monkeypatch): + """Toggling token_sizing via any settings path re-syncs the backend.""" + from lilbee.app.services import set_services + from lilbee.app.settings import _invalidate_caches + from tests.conftest import make_mock_services + + set_services(make_mock_services()) + try: + monkeypatch.setattr(cfg, "token_sizing", True) + reg, _unreg = self._patch_xberg(monkeypatch, listed=[]) + _invalidate_caches({"token_sizing"}) + reg.assert_called_once() + finally: + set_services(None) + + class TestServicesDataclass: def test_fields_are_immutable(self): from lilbee.app.services import CrawlerSyncState, Services diff --git a/tests/test_tokenizer_backend.py b/tests/test_tokenizer_backend.py new file mode 100644 index 000000000..b1b7be7b1 --- /dev/null +++ b/tests/test_tokenizer_backend.py @@ -0,0 +1,56 @@ +"""Tests for lilbee's xberg tokenizer backend (token-budgeted chunk sizing).""" + +from __future__ import annotations + +from lilbee.data.ingest.types import TokenizerBackendName +from lilbee.data.tokenizer_backend import LilbeeTokenizerBackend, _estimate_tokens + + +def test_name_is_lilbee(): + backend = LilbeeTokenizerBackend(count_fn=lambda _t: 1) + assert backend.name() == TokenizerBackendName.LILBEE + + +def test_initialize_and_shutdown_are_noops(): + backend = LilbeeTokenizerBackend(count_fn=lambda _t: 1) + assert backend.initialize() is None + assert backend.shutdown() is None + + +def test_exact_count_is_returned(): + backend = LilbeeTokenizerBackend(count_fn=lambda _t: 7) + assert backend.count_tokens("hello world") == 7 + + +def test_empty_text_is_zero_without_calling_count_fn(): + calls: list[str] = [] + + def count_fn(text: str) -> int: + calls.append(text) + return 5 + + backend = LilbeeTokenizerBackend(count_fn=count_fn) + assert backend.count_tokens("") == 0 + assert calls == [] + + +def test_exception_falls_back_to_char_estimate(): + def boom(_text: str) -> int: + raise RuntimeError("embedder unreachable") + + backend = LilbeeTokenizerBackend(count_fn=boom) + text = "x" * 9 + assert backend.count_tokens(text) == _estimate_tokens(text) + + +def test_zero_count_for_non_empty_text_falls_back_to_estimate(): + """xberg rejects a zero count for non-empty text; degrade to the estimate.""" + backend = LilbeeTokenizerBackend(count_fn=lambda _t: 0) + text = "x" * 9 + assert backend.count_tokens(text) == _estimate_tokens(text) + + +def test_estimate_is_conservative_and_at_least_one(): + assert _estimate_tokens("") == 1 + assert _estimate_tokens("abc") == 1 + assert _estimate_tokens("a" * 9) == 3 From 49dee2effc657ecc4b07ce6d59e84b4b54ff1300 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sun, 5 Jul 2026 19:48:38 -0400 Subject: [PATCH 41/77] fix: reconcile the local-model-api ingest/registry commits with rc9 Rebasing the tokenizer-sizing work onto the seven local-model-api commits that had landed on the branch surfaced pre-existing breakage in those commits, plus one rc9-API interaction. Bring the whole tree back to green (typecheck, lint, format, tests, 100% coverage): - vision_ocr_backend: type the OcrBackend callback against the public xberg.OcrConfig again -- rc9's Protocol uses it (kreuzberg-u4r), so the native-typed backend no longer conformed. Keeps the JSON-string backend_options handling; corrects the now-stale "native"/"is a dict" docstring. - extract: import OcrQualityThresholds under TYPE_CHECKING so the _forced_ocr_thresholds return annotation resolves. - placement: drop em-dashes from a docstring. - Refresh two stale tests to their commits' documented behavior: embed/rerank ctx is now the char budget (chunk_size * CHARS_PER_TOKEN + margin = 2056), and the /chat route's top_k default is None, not 0. - Cover the two untested paths those commits added: _forced_ocr_thresholds under LILBEE_OCR_FORCE, and the bare-repo split-set resolve that suppresses a failed first shard and skips the rest. --- src/lilbee/data/ingest/extract.py | 2 +- src/lilbee/data/ingest/vision_ocr_backend.py | 18 ++++++++-------- src/lilbee/providers/fleet/placement.py | 4 ++-- .../fleet/test_placement_from_spec.py | 4 +--- tests/test_fleet_planning.py | 11 +++++----- tests/test_ingest.py | 21 +++++++++++++++++++ tests/test_registry.py | 21 +++++++++++++++++++ tests/test_server_litestar.py | 4 ++-- tests/test_vision_ocr_backend.py | 4 +--- 9 files changed, 64 insertions(+), 25 deletions(-) diff --git a/src/lilbee/data/ingest/extract.py b/src/lilbee/data/ingest/extract.py index b7d7c8247..d217cef56 100644 --- a/src/lilbee/data/ingest/extract.py +++ b/src/lilbee/data/ingest/extract.py @@ -34,7 +34,7 @@ ) if TYPE_CHECKING: - from xberg import ExtractedDocument, ExtractionConfig, OcrConfig + from xberg import ExtractedDocument, ExtractionConfig, OcrConfig, OcrQualityThresholds log = logging.getLogger(__name__) diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py index 8cf83232b..90f380f25 100644 --- a/src/lilbee/data/ingest/vision_ocr_backend.py +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -17,13 +17,11 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator - from xberg import ExtractedDocument, OcrBackendType - - # The OcrBackend Protocol and the runtime trait callbacks use the native - # OcrConfig (xberg._xberg.OcrConfig), which is a different type from the public - # xberg.OcrConfig re-exported from xberg.options; a backend typed against the - # public one cannot satisfy the Protocol. Filed upstream (xberg). - from xberg._xberg import OcrConfig + # rc9's OcrBackend Protocol types the callback config as the public + # xberg.OcrConfig (kreuzberg-u4r); a backend typed against the native + # xberg._xberg.OcrConfig no longer satisfies it. The runtime object still + # arrives with backend_options as a JSON string (handled in _OcrConfigView). + from xberg import ExtractedDocument, OcrBackendType, OcrConfig # Token key inside OcrConfig.backend_options JSON. xberg does not propagate # contextvars into process_image (xberg-4w9), so per-request state travels as @@ -86,8 +84,10 @@ def backend_options_for(token: str) -> dict[str, str]: class _OcrConfigView: """Typed reader over the xberg OcrConfig object passed to process_image. - xberg hands the callback a native OcrConfig (alef typed trait callbacks), so - its fields are read directly as attributes; ``backend_options`` is a dict. + Typed against the public ``xberg.OcrConfig`` (rc9's OcrBackend Protocol), whose + fields are read directly as attributes. The native round-trip hands + ``backend_options`` back as a JSON string rather than the dict lilbee put in, so + ``request_token`` accepts both shapes. """ def __init__(self, config: OcrConfig) -> None: diff --git a/src/lilbee/providers/fleet/placement.py b/src/lilbee/providers/fleet/placement.py index 62c36ff2a..c699d8116 100644 --- a/src/lilbee/providers/fleet/placement.py +++ b/src/lilbee/providers/fleet/placement.py @@ -333,8 +333,8 @@ def placement_from_spec( def _derived_split(devices: tuple[int, ...], remaining: dict[int, float]) -> tuple[int, ...]: """Planner-style split for a spec entry without an explicit ``tensor_split``. - Each card's shard tracks its remaining usable VRAM (whole GiB, min 1) — the - same proportions the auto planner computes — so a card already carrying + Each card's shard tracks its remaining usable VRAM (whole GiB, min 1), the + same proportions the auto planner computes, so a card already carrying other roles takes a smaller share. An even split would charge every card the same shard and falsely reject layouts the planner itself serves (bb-lt7). """ diff --git a/tests/providers/fleet/test_placement_from_spec.py b/tests/providers/fleet/test_placement_from_spec.py index 71490d214..e0b22ff9c 100644 --- a/tests/providers/fleet/test_placement_from_spec.py +++ b/tests/providers/fleet/test_placement_from_spec.py @@ -117,9 +117,7 @@ def test_derives_planner_style_split_when_spec_has_none(): def test_explicit_tensor_split_still_wins(): # A spec that names its own split is honored verbatim, even when uneven. - spec = PlacementSpec( - {WorkerRole.CHAT: RolePlacement(devices=(0, 1), tensor_split=(3, 1))} - ) + spec = PlacementSpec({WorkerRole.CHAT: RolePlacement(devices=(0, 1), tensor_split=(3, 1))}) est = _proportional_peak({WorkerRole.CHAT: 40}) placement = placement_from_spec( spec, (WorkerRole.CHAT,), {0: 80 * GIB, 1: 80 * GIB}, estimate_peak=est diff --git a/tests/test_fleet_planning.py b/tests/test_fleet_planning.py index afc92376f..85f6d23f5 100644 --- a/tests/test_fleet_planning.py +++ b/tests/test_fleet_planning.py @@ -85,16 +85,17 @@ def test_role_ctx_chat_uses_dynamic_picker_when_unset(monkeypatch) -> None: def test_role_ctx_embed_covers_chunk_size_plus_margin(monkeypatch) -> None: - # A 32K-trained embedder is sized to the chunk length plus the truncation margin - # (so a full chunk_size input is not truncated), not its full context, so its - # placement estimate doesn't balloon (200GB+) and starve the role alongside a giant. + # A 32K-trained embedder is sized to the chunker's character budget plus the + # truncation margin (chunk_size * CHARS_PER_TOKEN + margin -- the provable token + # ceiling for any chunk), not its full context, so its placement estimate doesn't + # balloon (200GB+) and starve the role alongside a giant. monkeypatch.setattr( "lilbee.providers.engine_params.train_ctx_from_meta", lambda _meta, *, fallback, model_path: 32768, ) monkeypatch.setattr(cfg, "chunk_size", 512) - assert planning_mod._role_ctx(WorkerRole.EMBED, Path("/m/e.gguf"), {}) == 520 - assert planning_mod._role_ctx(WorkerRole.RERANK, Path("/m/r.gguf"), {}) == 520 + assert planning_mod._role_ctx(WorkerRole.EMBED, Path("/m/e.gguf"), {}) == 2056 + assert planning_mod._role_ctx(WorkerRole.RERANK, Path("/m/r.gguf"), {}) == 2056 def test_embed_ctx_token_cap_fits_full_chunk(monkeypatch) -> None: diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 833621d63..22d8236d1 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -828,6 +828,27 @@ async def test_cancel_preserves_old_chunks_for_modified_file( mock_svc.store.write_chunks_batch.assert_not_called() +class TestForcedOcrThresholds: + """cfg-independent LILBEE_OCR_FORCE lever for scans with a garbage text layer.""" + + def test_none_when_env_unset(self, monkeypatch): + from lilbee.data.ingest.extract import _forced_ocr_thresholds + + monkeypatch.delenv("LILBEE_OCR_FORCE", raising=False) + assert _forced_ocr_thresholds() is None + + @pytest.mark.parametrize("value", ["1", "true", "YES"]) + def test_impossible_floor_when_env_set(self, monkeypatch, value): + from lilbee.data.ingest.extract import _forced_ocr_thresholds + + monkeypatch.setenv("LILBEE_OCR_FORCE", value) + thresholds = _forced_ocr_thresholds() + assert thresholds is not None + # An unreachable non-whitespace floor fails every page's text-layer gate, + # forcing OCR on scans whose garbage text layer would otherwise skip it. + assert thresholds.min_total_non_whitespace == 10**9 + + class TestIngestHelpers: """Cover edge cases in ingest_document and ingest_code_sync.""" diff --git a/tests/test_registry.py b/tests/test_registry.py index 2fb389750..b2b1b1c19 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -470,6 +470,27 @@ def test_bare_repo_recovers_subdir_split_gguf_from_raw_cache(self, tmp_path: Pat for n in (2, 3): assert (resolved.parent / f"MiniMax-M2-Q4_K_M-0000{n}-of-00003.gguf").exists() + def test_bare_repo_split_that_cannot_resolve_skips_shards_and_raises( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A bare repo whose only cached GGUFs are an unresolvable split set (a shard + missing) suppresses shard 1's failure, skips the trailing shards, and ends in + 'not installed' rather than handing back a partial resolve.""" + registry = ModelRegistry(tmp_path) + repo = "ggml-org/gpt-oss-120b-GGUF" + for n in (1, 2, 3): + _seed_hf_cache( + tmp_path, + repo=repo, + filename=f"m-mxfp4-0000{n}-of-00003.gguf", + content=f"shard-{n}".encode(), + ) + # Every shard reports missing, so _resolve_split raises on shard 1 (suppressed); + # shards 2 and 3 are skipped as non-first, and the loop exhausts unresolved. + monkeypatch.setattr(registry, "_split_shards_present", lambda *_a, **_k: False) + with pytest.raises(KeyError, match="not installed"): + registry.resolve(repo) + def test_shard_paths_returns_every_split_shard(self, tmp_path: Path) -> None: registry = ModelRegistry(tmp_path) repo = "ggml-org/gpt-oss-120b-GGUF" diff --git a/tests/test_server_litestar.py b/tests/test_server_litestar.py index 63ee560f3..caeeb30d3 100644 --- a/tests/test_server_litestar.py +++ b/tests/test_server_litestar.py @@ -286,7 +286,7 @@ def test_forwards_history(self, mock_chat, client): resp = client.post("/api/chat", json={"question": "q", "history": history}) assert resp.status_code == 201 mock_chat.assert_awaited_once_with( - question="q", history=history, top_k=0, options=None, chunk_type=None + question="q", history=history, top_k=None, options=None, chunk_type=None ) @mock.patch( @@ -297,7 +297,7 @@ def test_forwards_history(self, mock_chat, client): def test_default_empty_history(self, mock_chat, client): client.post("/api/chat", json={"question": "q"}) mock_chat.assert_awaited_once_with( - question="q", history=[], top_k=0, options=None, chunk_type=None + question="q", history=[], top_k=None, options=None, chunk_type=None ) @mock.patch("lilbee.server.handlers.chat", new_callable=AsyncMock) diff --git a/tests/test_vision_ocr_backend.py b/tests/test_vision_ocr_backend.py index 47f9bdf02..7859d2ca8 100644 --- a/tests/test_vision_ocr_backend.py +++ b/tests/test_vision_ocr_backend.py @@ -136,9 +136,7 @@ def test_json_string_backend_options_resolve(self): ticks: list[int] = [] be, calls = _backend() with ocr_request(on_page=lambda: ticks.append(1), timeout=7.5) as token: - be.process_image( - b"PNG", _cfg(backend_options=json.dumps(backend_options_for(token))) - ) + be.process_image(b"PNG", _cfg(backend_options=json.dumps(backend_options_for(token)))) assert calls[0][3] == 7.5 assert ticks == [1] From d52e6311849649282f8c3c9a71cb7662c1e3a214 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Wed, 8 Jul 2026 02:03:31 -0400 Subject: [PATCH 42/77] fix(ingest): scale the extraction offload pool with vCPU count The dedicated ingest thread pool was capped at min(32, cpu+4). On a many-vCPU OCR box the extraction stage (PDF rasterization + OCR) runs on this pool, so 32 threads left most cores idle and pinned full-corpus throughput below the vision fleet's slot capacity -- GPUs sat under 50% util while load swung between 20 and 159. Size the pool to cpu_count + 4 and add a LILBEE_INGEST_MAX_WORKERS override to tune a specific box, matching the existing LILBEE_CPU_QUOTA idiom. (cherry picked from commit 732d63d20c6e43b48adb2cee889259c99b33180c) --- src/lilbee/data/ingest/offload.py | 31 +++++++++++++++++++-- tests/test_ingest_offload.py | 46 +++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 tests/test_ingest_offload.py diff --git a/src/lilbee/data/ingest/offload.py b/src/lilbee/data/ingest/offload.py index 80befe674..c7165f855 100644 --- a/src/lilbee/data/ingest/offload.py +++ b/src/lilbee/data/ingest/offload.py @@ -11,18 +11,45 @@ import asyncio import contextvars import functools +import logging import os from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from typing import ParamSpec, TypeVar +log = logging.getLogger(__name__) + _P = ParamSpec("_P") _R = TypeVar("_R") +_MAX_WORKERS_ENV = "LILBEE_INGEST_MAX_WORKERS" + def _max_workers() -> int: - """Mirror the default-executor sizing; the point is isolation, not tuning.""" - return min(32, (os.cpu_count() or 4) + 4) + """Concurrent slots for ingest offload work; honors ``LILBEE_INGEST_MAX_WORKERS``. + + Extraction rasterizes PDFs and drives OCR on this pool, so its width caps how + many documents feed the GPU OCR/embed slots at once. The old fixed ``32`` + ceiling left most cores idle on a many-vCPU box and pinned full-corpus + throughput below the vision fleet's capacity, so the default now scales with + the vCPU count (``os.cpu_count() + 4``, the ``+4`` headroom for threads + parked on OCR/embed I/O). The override accepts a positive integer; non-positive + or unparseable values fall back to the default and a warning is logged. + """ + override = os.environ.get(_MAX_WORKERS_ENV) + if override is not None: + try: + value = int(override) + if value > 0: + return value + except ValueError: + pass # bad override falls through to the warning + default below + log.warning( + "Ignoring %s=%r: must be a positive integer; using default.", + _MAX_WORKERS_ENV, + override, + ) + return (os.cpu_count() or 4) + 4 @functools.cache diff --git a/tests/test_ingest_offload.py b/tests/test_ingest_offload.py new file mode 100644 index 000000000..819b0ca77 --- /dev/null +++ b/tests/test_ingest_offload.py @@ -0,0 +1,46 @@ +"""Sizing of the dedicated ingest offload pool. + +Extraction (PDF rasterization + OCR) runs on this pool, so its width caps how +many documents rasterize in parallel to feed the GPU OCR/embed slots on a +full-corpus run. These lock in that the width scales with the vCPU count and is +operator-overridable, rather than the old fixed 32-thread ceiling. +""" + +from __future__ import annotations + +import pytest + +from lilbee.data.ingest import offload + + +@pytest.fixture(autouse=True) +def _clear_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LILBEE_INGEST_MAX_WORKERS", raising=False) + + +def test_default_scales_past_the_old_32_ceiling(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(offload.os, "cpu_count", lambda: 128) + assert offload._max_workers() == 132 + + +def test_default_keeps_headroom_on_a_small_box(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(offload.os, "cpu_count", lambda: 8) + assert offload._max_workers() == 12 + + +def test_default_falls_back_when_cpu_count_is_unknown(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(offload.os, "cpu_count", lambda: None) + assert offload._max_workers() == 8 + + +def test_env_override_wins_over_the_scaled_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(offload.os, "cpu_count", lambda: 8) + monkeypatch.setenv("LILBEE_INGEST_MAX_WORKERS", "200") + assert offload._max_workers() == 200 + + +@pytest.mark.parametrize("bad", ["nan", "0", "-4", " "]) +def test_invalid_env_override_is_ignored(monkeypatch: pytest.MonkeyPatch, bad: str) -> None: + monkeypatch.setattr(offload.os, "cpu_count", lambda: 8) + monkeypatch.setenv("LILBEE_INGEST_MAX_WORKERS", bad) + assert offload._max_workers() == 12 From bd0f720fd01e26b65742a88f4776c47754a5273c Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Wed, 8 Jul 2026 03:25:45 -0400 Subject: [PATCH 43/77] feat(ingest): adaptive concurrency that hill-climbs to the hardware's throughput knee Adds a selectable ingest concurrency mode via LILBEE_INGEST_CONCURRENCY: - static (default): the existing fixed semaphore over the cpu+4 pool, unchanged. - adaptive-conservative / adaptive-aggressive: a background controller resizes the extraction-admission limit toward this box's real throughput knee instead of a hardcoded utilization target. The control law is a safety-gated AIMD hill-climb on smoothed per-page throughput (AIMD, BBR/Kleinrock, the CLR ThreadPool hill-climber, the Universal Scalability Law). It climbs while pages-per-second still improves and settles at the knee; a Little's-Law residence-time estimate gives a TCP-Vegas-style latency-gradient veto that catches the knee before throughput visibly rolls over. Hard guardrails on CPU, free RAM, and GPU temperature force an immediate multiplicative backoff, so it cannot thrash the CPU, exhaust memory, or overheat a GPU. Adaptive engages only when a GPU fleet is present; a GPU-less host always uses static, and the tuner is best-effort so it can never crash the ingest it advises. (cherry picked from commit 9d4cfb85e8e84d51be8f2cac6bba13b8c4595147) --- src/lilbee/data/ingest/adaptive.py | 419 +++++++++++++++++++++++++++++ src/lilbee/data/ingest/offload.py | 9 + src/lilbee/data/ingest/pipeline.py | 124 ++++++--- tests/test_ingest.py | 18 ++ tests/test_ingest_adaptive.py | 395 +++++++++++++++++++++++++++ 5 files changed, 929 insertions(+), 36 deletions(-) create mode 100644 src/lilbee/data/ingest/adaptive.py create mode 100644 tests/test_ingest_adaptive.py diff --git a/src/lilbee/data/ingest/adaptive.py b/src/lilbee/data/ingest/adaptive.py new file mode 100644 index 000000000..f46c3c43e --- /dev/null +++ b/src/lilbee/data/ingest/adaptive.py @@ -0,0 +1,419 @@ +"""Adaptive ingest concurrency: hill-climb to the hardware's throughput knee. + +The extraction-admission limit (how many documents are in their compute phase at +once) can run in one of three modes, chosen by ``LILBEE_INGEST_CONCURRENCY``: + +- ``static`` -- a fixed limit (the pipeline's ``_max_concurrent()``); the proven + default and the guaranteed-safe fallback. +- ``adaptive-conservative`` / ``adaptive-aggressive`` -- a background controller + resizes the limit every few seconds, climbing while throughput still improves + and backing off the instant a safety signal (CPU, free RAM, GPU temperature) + says the box is under pressure. + +The control law is a safety-gated AIMD hill-climb on smoothed throughput. The +only thing that pushes the limit up is throughput still improving, so the fixed +point is *this box's* real operating knee rather than a hardcoded utilization +target (BBR / Kleinrock). GPU utilization is used only as a saturation veto, never +as a setpoint. Multiplicative decrease on any danger signal is AIMD's proven +fast-safe retreat; EWMA smoothing plus an asymmetric dead band and a one-step slew +limit keep it from oscillating (CLR ThreadPool hill-climbing, TCP Vegas). + +``decide`` is a pure function -- the whole control law with no clock, asyncio, or +hardware -- so the policy is unit-tested with plain numbers. ``ResizableGate`` and +``AdaptiveController`` are the async machinery around it. +""" + +from __future__ import annotations + +import asyncio +import logging +import math +import os +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from enum import StrEnum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from lilbee.providers.fleet.gpu_stats import DeviceLike + +log = logging.getLogger(__name__) + +_MODE_ENV = "LILBEE_INGEST_CONCURRENCY" + + +class ConcurrencyMode(StrEnum): + """How the extraction-admission limit is chosen for a sync run.""" + + STATIC = "static" + ADAPTIVE_CONSERVATIVE = "adaptive-conservative" + ADAPTIVE_AGGRESSIVE = "adaptive-aggressive" + + +def resolve_mode() -> ConcurrencyMode: + """The concurrency mode from ``LILBEE_INGEST_CONCURRENCY``; ``static`` by default. + + An unset or unrecognized value falls back to ``static`` (a warning is logged for + an unrecognized one), so a typo can never silently enable adaptive control. + """ + raw = os.environ.get(_MODE_ENV, "").strip().lower() + if not raw: + return ConcurrencyMode.STATIC + try: + return ConcurrencyMode(raw) + except ValueError: + log.warning( + "Ignoring %s=%r: expected one of %s; using %s.", + _MODE_ENV, + raw, + [m.value for m in ConcurrencyMode], + ConcurrencyMode.STATIC.value, + ) + return ConcurrencyMode.STATIC + + +@dataclass(frozen=True) +class SafetyLimits: + """Hard guardrails, identical across adaptive profiles -- safety is never relaxed. + + Crossing a ``_crit`` line forces an immediate multiplicative backoff; a ``_warn`` + / ``_soft`` line only vetoes further increases. Temperature and RAM are read raw + (never smoothed -- a fire alarm should not be averaged away). + """ + + gpu_sat_pct: float = 97.0 + cpu_soft_pct: float = 90.0 + cpu_crit_pct: float = 97.0 + ram_soft_free: float = 0.20 + ram_min_free: float = 0.10 + temp_warn_c: float = 80.0 + temp_crit_c: float = 85.0 + decrease_factor: float = 0.5 + + +@dataclass(frozen=True) +class ConcurrencyProfile: + """Climb-speed parameters for one adaptive profile (safety limits are shared).""" + + name: str + interval_s: float + ewma_gamma: float + deadband_frac: float + cool_down_intervals: int + sqrt_step: bool + latency_veto_ratio: float # veto increases once residence time inflates past baseline x this + safety: SafetyLimits = SafetyLimits() + + def step(self, permits: int) -> int: + """Additive step size at the current limit; larger early when ``sqrt_step``.""" + if self.sqrt_step: + return 1 + math.isqrt(max(0, permits)) // 4 + return 1 + + +CONSERVATIVE = ConcurrencyProfile( + name="conservative", + interval_s=5.0, + ewma_gamma=0.3, + deadband_frac=0.05, + cool_down_intervals=3, + sqrt_step=False, + latency_veto_ratio=1.5, +) +AGGRESSIVE = ConcurrencyProfile( + name="aggressive", + interval_s=2.0, + ewma_gamma=0.5, + deadband_frac=0.03, + cool_down_intervals=2, + sqrt_step=True, + latency_veto_ratio=2.0, +) + + +def profile_for(mode: ConcurrencyMode) -> ConcurrencyProfile | None: + """The profile for an adaptive mode, or None for ``static``.""" + return { + ConcurrencyMode.ADAPTIVE_CONSERVATIVE: CONSERVATIVE, + ConcurrencyMode.ADAPTIVE_AGGRESSIVE: AGGRESSIVE, + }.get(mode) + + +@dataclass(frozen=True) +class Signals: + """One tick's measured state. ``gpu_*`` are None when no GPU telemetry is available.""" + + throughput: float # OCR pages completed since the previous tick (work done, not doc count) + gpu_util_pct: float | None + gpu_temp_c: float | None + cpu_pct: float + ram_free_frac: float + + +@dataclass(frozen=True) +class ControllerState: + """The controller's carried state between ticks.""" + + permits: int + ewma_tput: float | None # smoothed throughput from the previous tick; None at start + direction: int # +1 climbing, -1 backing off -- the last hill-climb step's sign + cool_down: int # intervals remaining during which increases are suppressed + w_min: float | None = None # smallest observed residence-time estimate (Little's Law baseline) + + +def _is_critical(signals: Signals, s: SafetyLimits) -> bool: + """A signal that demands an immediate hard backoff (thermal / OOM / CPU meltdown).""" + temp = signals.gpu_temp_c + return ( + (temp is not None and temp >= s.temp_crit_c) + or signals.ram_free_frac <= s.ram_min_free + or signals.cpu_pct >= s.cpu_crit_pct + ) + + +def _increase_vetoed( + profile: ConcurrencyProfile, + signals: Signals, + *, + cool_down: int, + gpu_saturated: bool, + w_est: float | None, + w_min: float | None, +) -> bool: + """Whether soft pressure, saturation, cool-down, or inflating latency blocks a climb.""" + s = profile.safety + temp = signals.gpu_temp_c + latency_inflated = ( + w_est is not None and w_min is not None and w_est > w_min * profile.latency_veto_ratio + ) + return ( + cool_down > 0 + or signals.cpu_pct >= s.cpu_soft_pct + or signals.ram_free_frac < s.ram_soft_free + or (temp is not None and temp >= s.temp_warn_c) + or gpu_saturated + or latency_inflated + ) + + +def _hill_climb( + profile: ConcurrencyProfile, + permits: int, + direction: int, + delta: float | None, + new_ewma: float, + clamp: Callable[[int], int], +) -> tuple[int, int]: + """One dead-banded hill-climb step; returns the next (permits, direction). + + No baseline yet -> a gentle probe up; a clear gain -> step in the same direction; + a clear loss -> reverse; inside the dead band -> hold. + """ + if delta is None: + return clamp(permits + profile.step(permits)), 1 + band = profile.deadband_frac * (new_ewma if new_ewma > 0 else 1.0) + if delta > band: + return clamp(permits + direction * profile.step(permits)), direction + if delta < -band: + direction = -direction + return clamp(permits + direction * profile.step(permits)), direction + return permits, direction + + +def decide( + profile: ConcurrencyProfile, + state: ControllerState, + signals: Signals, + permit_min: int, + permit_max: int, +) -> ControllerState: + """Pure control law: fold one tick of signals into the next permit target. + + Priority: (1) any critical safety signal forces a multiplicative decrease and a + cool-down; (2) a soft-pressure, GPU-saturation, or latency-gradient signal vetoes + increases (with a single additive decrease when GPUs are saturated *and* throughput + is falling -- the Universal Scalability Law's retrograde region); (3) otherwise + hill-climb toward the knee. Always clamped to ``[permit_min, permit_max]``. + + The latency veto is the leading knee indicator (TCP Vegas / Netflix Gradient2): + residence time ``W`` is estimated by Little's Law as ``permits / throughput``; once + ``W`` inflates past its observed minimum (the unloaded baseline) by the profile's + ratio, work is queueing at the bottleneck and increases stop -- before throughput + visibly rolls over. + """ + if state.ewma_tput is None: + new_ewma: float = signals.throughput + delta: float | None = None + else: + gamma = profile.ewma_gamma + new_ewma = gamma * signals.throughput + (1.0 - gamma) * state.ewma_tput + delta = new_ewma - state.ewma_tput + + permits = state.permits + cool_down = max(0, state.cool_down - 1) + + # Little's Law residence-time estimate and its running minimum (the latency baseline). + w_est = permits / new_ewma if new_ewma > 0 else None + w_min = state.w_min if w_est is None else min(state.w_min or w_est, w_est) + + def clamp(p: int) -> int: + return max(permit_min, min(permit_max, p)) + + def out(new_permits: int, new_direction: int, new_cool_down: int) -> ControllerState: + return ControllerState(new_permits, new_ewma, new_direction, new_cool_down, w_min) + + if _is_critical(signals, profile.safety): + backoff = clamp(int(permits * profile.safety.decrease_factor)) + return out(backoff, -1, profile.cool_down_intervals) + + util = signals.gpu_util_pct + gpu_saturated = util is not None and util >= profile.safety.gpu_sat_pct + if gpu_saturated and delta is not None and delta < 0: + return out(clamp(permits - profile.step(permits)), -1, cool_down) # USL retrograde + + if _increase_vetoed( + profile, signals, cool_down=cool_down, gpu_saturated=gpu_saturated, w_est=w_est, w_min=w_min + ): + return out(permits, state.direction, cool_down) + + permits, direction = _hill_climb(profile, permits, state.direction, delta, new_ewma, clamp) + return out(permits, direction, cool_down) + + +class ResizableGate: + """An async admission gate whose permit ceiling can change while it is in use. + + Same ``async with`` shape as ``asyncio.Semaphore``, plus ``set_limit``: growing + wakes blocked acquirers, shrinking lowers the ceiling and lets the surplus drain + as active holders release. The limit never drops below one, so a shrink can never + deadlock a run. + """ + + def __init__(self, limit: int) -> None: + self._limit = max(1, limit) + self._active = 0 + self._cond = asyncio.Condition() + + @property + def limit(self) -> int: + return self._limit + + async def acquire(self) -> None: + async with self._cond: + await self._cond.wait_for(lambda: self._active < self._limit) + self._active += 1 + + async def release(self) -> None: + async with self._cond: + self._active -= 1 + self._cond.notify_all() + + async def __aenter__(self) -> ResizableGate: + await self.acquire() + return self + + async def __aexit__(self, *exc: object) -> None: + await self.release() + + async def set_limit(self, new_limit: int) -> None: + async with self._cond: + self._limit = max(1, new_limit) + self._cond.notify_all() + + +class AdaptiveController: + """Drives a :class:`ResizableGate`'s limit from live signals until cancelled. + + ``sample(throughput)`` returns the current :class:`Signals`; ``completed()`` is a + monotonic count of finished documents, from which per-interval throughput is + derived. Both are injected so the controller runs in tests with no clock or GPU. + """ + + def __init__( + self, + gate: ResizableGate, + profile: ConcurrencyProfile, + sample: Callable[[float], Signals], + completed: Callable[[], int], + *, + permit_min: int, + permit_max: int, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + self._gate = gate + self._profile = profile + self._sample = sample + self._completed = completed + self._permit_min = permit_min + self._permit_max = permit_max + self._sleep = sleep + + async def run(self) -> None: + """Sample-decide-resize loop. Runs until the task is cancelled. + + A tuning tick is best-effort: a transient sampling/probe failure is logged and + skipped, never propagated, so the controller can never crash the ingest it is + only advising. Cancellation (``CancelledError``) still ends the loop cleanly. + """ + state = ControllerState(self._gate.limit, None, 1, 0) + last_completed = self._completed() + while True: + await self._sleep(self._profile.interval_s) + try: + now_completed = self._completed() + throughput = float(max(0, now_completed - last_completed)) + last_completed = now_completed + state = decide( + self._profile, + state, + self._sample(throughput), + self._permit_min, + self._permit_max, + ) + if state.permits != self._gate.limit: + await self._gate.set_limit(state.permits) + log.debug("adaptive ingest: limit -> %d", state.permits) + except Exception: + log.debug("adaptive ingest: tuning tick failed; skipping", exc_info=True) + + +def enumerate_fleet_devices() -> Sequence[DeviceLike]: + """The GPU devices to read telemetry from, or empty when none can be enumerated. + + Any failure (no engine binary, probe error) degrades to an empty list, which the + caller treats as "no fleet to feed" and falls back to the static limit. + """ + try: + from lilbee.providers.fleet.binary import resolve_llama_server + from lilbee.providers.fleet.planning import resolve_devices + + return resolve_devices(resolve_llama_server()) + except Exception: + log.debug("adaptive ingest: device enumeration failed; using static limit", exc_info=True) + return [] + + +def make_signal_sampler(devices: Sequence[DeviceLike]) -> Callable[[float], Signals]: + """Build a sampler that reads mean GPU util, max GPU temp, CPU %, and free RAM. + + GPU util/temp are None when no device reports them (the controller then relies on + the CPU and RAM guards alone); throughput is supplied by the controller. + """ + import psutil + + from lilbee.providers.fleet.gpu_stats import probe_gpu_stats + + def sample(throughput: float) -> Signals: + stats = probe_gpu_stats(devices) + utils = [g.utilization_pct for g in stats.values() if g.utilization_pct is not None] + temps = [g.temperature_c for g in stats.values() if g.temperature_c is not None] + vm = psutil.virtual_memory() + return Signals( + throughput=throughput, + gpu_util_pct=(sum(utils) / len(utils)) if utils else None, + gpu_temp_c=float(max(temps)) if temps else None, + cpu_pct=psutil.cpu_percent(interval=None), + ram_free_frac=vm.available / vm.total, # psutil total is always positive + ) + + return sample diff --git a/src/lilbee/data/ingest/offload.py b/src/lilbee/data/ingest/offload.py index c7165f855..977d31050 100644 --- a/src/lilbee/data/ingest/offload.py +++ b/src/lilbee/data/ingest/offload.py @@ -52,6 +52,15 @@ def _max_workers() -> int: return (os.cpu_count() or 4) + 4 +def max_workers() -> int: + """The ingest pool's worker count -- its hard concurrency ceiling. + + The adaptive-concurrency controller uses this as the upper bound on in-flight + documents, since each one needs a pool thread to run its blocking extraction. + """ + return _max_workers() + + @functools.cache def _ingest_executor() -> ThreadPoolExecutor: """The shared ingest pool, created on first use (cache makes it a singleton).""" diff --git a/src/lilbee/data/ingest/pipeline.py b/src/lilbee/data/ingest/pipeline.py index 4032a386b..fd274f890 100644 --- a/src/lilbee/data/ingest/pipeline.py +++ b/src/lilbee/data/ingest/pipeline.py @@ -23,10 +23,18 @@ from lilbee.app.services import get_services from lilbee.core.config import active_config +from lilbee.data.ingest.adaptive import ( + AdaptiveController, + ResizableGate, + enumerate_fleet_devices, + make_signal_sampler, + profile_for, + resolve_mode, +) from lilbee.data.ingest.code import ingest_code_sync from lilbee.data.ingest.discovery import classify_file, discover_files, file_hash from lilbee.data.ingest.extract import ingest_document, ingest_markdown -from lilbee.data.ingest.offload import to_ingest_thread +from lilbee.data.ingest.offload import max_workers, to_ingest_thread from lilbee.data.ingest.skip_marker import ( clear_skip_markers, load_skip_markers, @@ -518,6 +526,37 @@ def _callback(event_type: EventType, data: ProgressEvent) -> None: _TASK_WINDOW_MULTIPLIER = 2 +def _build_admission( + baseline: int, pages_done: list[int] +) -> tuple[asyncio.Semaphore | ResizableGate, int, asyncio.Task[None] | None]: + """The batch's admission control, plus its task-window size and controller task. + + Static mode (the default) returns a fixed semaphore and no controller. Adaptive + mode, when a GPU fleet is present to feed, returns a resizable gate and a running + :class:`AdaptiveController` that tunes it toward this box's throughput knee; with + no fleet it falls back to the static path so a GPU-less host is never affected. + """ + profile = profile_for(resolve_mode()) + devices = enumerate_fleet_devices() if profile is not None else [] + if profile is None or not devices: + return asyncio.Semaphore(baseline), baseline * _TASK_WINDOW_MULTIPLIER, None + permit_max = max_workers() + gate = ResizableGate(min(baseline, permit_max)) + controller = AdaptiveController( + gate, + profile, + make_signal_sampler(devices), + lambda: pages_done[0], + permit_min=1, + permit_max=permit_max, + ) + task = asyncio.ensure_future(controller.run()) + log.info( + "Adaptive ingest concurrency (%s): start %d, max %d", profile.name, gate.limit, permit_max + ) + return gate, permit_max * _TASK_WINDOW_MULTIPLIER, task + + async def ingest_batch( files_to_process: list[FileToProcess], added: dict[str, None], @@ -539,13 +578,16 @@ async def ingest_batch( # Honor LILBEE_INGEST_TRACE once per batch: it raises the trace loggers above # the default WARNING so per-file extraction lines actually surface. configure_trace_from_env() - semaphore = asyncio.Semaphore(_max_concurrent()) - window = _max_concurrent() * _TASK_WINDOW_MULTIPLIER + # Throughput is measured in OCR pages, not documents: a document's cost scales + # with its page count (a 500-page scan is 500x a memo), so pages are the unbiased + # unit of GPU-feeding work for the adaptive controller to hill-climb on. + pages_done = [0] + admission, window, controller_task = _build_admission(_max_concurrent(), pages_done) total_files = len(files_to_process) async def _process_one(entry: FileToProcess, file_index: int) -> _IngestResult: name = entry.name - async with semaphore: + async with admission: if cancel and cancel.is_set(): raise asyncio.CancelledError @@ -576,6 +618,7 @@ async def _process_one(entry: FileToProcess, file_index: int) -> _IngestResult: EventType.FILE_DONE, FileDoneEvent(file=name, status="ok", chunks=len(records)), ) + pages_done[0] += max(1, len(page_texts)) # OCR pages cleared: the throughput signal return _IngestResult( name, entry.path, @@ -611,38 +654,12 @@ async def _process_one(entry: FileToProcess, file_index: int) -> _IngestResult: EventType.FILE_DONE, FileDoneEvent(file=name, status="error", chunks=0), ) + pages_done[0] += 1 # cleared the gate (as a failure); still a throughput tick return _IngestResult(name, entry.path, 0, error=exc) pending = (_process_one(entry, idx) for idx, entry in enumerate(files_to_process, 1)) - if quiet: - await _collect_results( - pending, - total_files, - added, - updated, - failed, - skipped, - window=window, - on_progress=on_progress, - flush_failed=flush_failed, - reasons=reasons, - ) - else: - with Progress( - SpinnerColumn(), - TextColumn("{task.description}"), - BarColumn(), - MofNCompleteColumn(), - TimeElapsedColumn(), - transient=True, - ) as progress: - ptask = progress.add_task("Ingesting documents...", total=total_files) - # The bar advances once per file (in _collect_results), so a single - # multi-page scanned PDF would freeze at "0/1" through its whole - # OCR + embed phase. Drive the spinner's description off the same - # EXTRACT (OCR page i/N) and EMBED (chunk i/N) events the TUI uses - # so the row visibly moves while one file is being worked. - phase_progress = _phase_progress_callback(progress, ptask, on_progress) + try: + if quiet: await _collect_results( pending, total_files, @@ -651,12 +668,47 @@ async def _process_one(entry: FileToProcess, file_index: int) -> _IngestResult: failed, skipped, window=window, - on_progress=phase_progress, - progress=progress, - ptask=ptask, + on_progress=on_progress, flush_failed=flush_failed, reasons=reasons, ) + else: + with Progress( + SpinnerColumn(), + TextColumn("{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TimeElapsedColumn(), + transient=True, + ) as progress: + ptask = progress.add_task("Ingesting documents...", total=total_files) + # The bar advances once per file (in _collect_results), so a single + # multi-page scanned PDF would freeze at "0/1" through its whole + # OCR + embed phase. Drive the spinner's description off the same + # EXTRACT (OCR page i/N) and EMBED (chunk i/N) events the TUI uses + # so the row visibly moves while one file is being worked. + phase_progress = _phase_progress_callback(progress, ptask, on_progress) + await _collect_results( + pending, + total_files, + added, + updated, + failed, + skipped, + window=window, + on_progress=phase_progress, + progress=progress, + ptask=ptask, + flush_failed=flush_failed, + reasons=reasons, + ) + finally: + # Stop the adaptive controller (if any) before returning: its background + # loop must not outlive the batch it was tuning. + if controller_task is not None: + controller_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await controller_task # Accumulate roughly this many chunks across documents before one batched diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 22d8236d1..0b9bec49c 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -197,6 +197,24 @@ async def test_ingest_text_file(self, mock_extract_file, isolated_env): mock_extract_file.assert_called() assert any("test.txt" in str(call) for call in mock_extract_file.call_args_list) + async def test_adaptive_mode_ingests_and_stops_controller( + self, mock_extract_file, isolated_env, monkeypatch + ): + (isolated_env / "adaptive.txt").write_text("Adaptive-mode content for ingest.") + from lilbee.data.ingest import pipeline, sync + from lilbee.data.ingest.adaptive import Signals + + monkeypatch.setenv("LILBEE_INGEST_CONCURRENCY", "adaptive-conservative") + # A fake fleet + sampler so the adaptive path engages without real GPU probing. + monkeypatch.setattr(pipeline, "enumerate_fleet_devices", lambda: [object()]) + monkeypatch.setattr( + pipeline, + "make_signal_sampler", + lambda _devices: (lambda t: Signals(t, 50.0, 60.0, 50.0, 0.5)), + ) + result = await sync(quiet=True) + assert "adaptive.txt" in result.added + async def test_quiet_mode_suppresses_progress(self, mock_extract_file, isolated_env): (isolated_env / "quiet.txt").write_text("Quiet mode test content.") from lilbee.data.ingest import sync diff --git a/tests/test_ingest_adaptive.py b/tests/test_ingest_adaptive.py new file mode 100644 index 000000000..e5e36384f --- /dev/null +++ b/tests/test_ingest_adaptive.py @@ -0,0 +1,395 @@ +"""Adaptive ingest concurrency: mode resolution, the pure control law, the +resizable gate, and the controller loop. + +The control law (:func:`decide`) is exercised with plain numbers -- no clock, no +asyncio, no GPU -- since it is a pure function; the gate and controller get small +asyncio tests with injected signals so they run without hardware. +""" + +from __future__ import annotations + +import asyncio +import contextlib + +import pytest + +from lilbee.data.ingest.adaptive import ( + AGGRESSIVE, + CONSERVATIVE, + AdaptiveController, + ConcurrencyMode, + ControllerState, + ResizableGate, + Signals, + decide, + enumerate_fleet_devices, + make_signal_sampler, + profile_for, + resolve_mode, +) + +MIN, MAX = 1, 100 + + +def _signals(throughput: float, **over: float | None) -> Signals: + """A tick with every safety signal comfortable, overridable per test.""" + base: dict[str, float | None] = { + "gpu_util_pct": 50.0, + "gpu_temp_c": 60.0, + "cpu_pct": 50.0, + "ram_free_frac": 0.5, + } + base.update(over) + return Signals(throughput=throughput, **base) # type: ignore[arg-type] + + +def _state( + permits: int, + ewma: float | None = 100.0, + direction: int = 1, + cool_down: int = 0, + w_min: float | None = None, +): + return ControllerState( + permits=permits, ewma_tput=ewma, direction=direction, cool_down=cool_down, w_min=w_min + ) + + +# --- mode resolution ------------------------------------------------------- + + +def test_mode_defaults_to_static(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LILBEE_INGEST_CONCURRENCY", raising=False) + assert resolve_mode() is ConcurrencyMode.STATIC + + +@pytest.mark.parametrize( + ("value", "mode"), + [ + ("static", ConcurrencyMode.STATIC), + ("adaptive-conservative", ConcurrencyMode.ADAPTIVE_CONSERVATIVE), + ("ADAPTIVE-AGGRESSIVE", ConcurrencyMode.ADAPTIVE_AGGRESSIVE), + ], +) +def test_mode_parses_known_values(monkeypatch: pytest.MonkeyPatch, value: str, mode) -> None: + monkeypatch.setenv("LILBEE_INGEST_CONCURRENCY", value) + assert resolve_mode() is mode + + +def test_unknown_mode_falls_back_to_static(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LILBEE_INGEST_CONCURRENCY", "turbo") + assert resolve_mode() is ConcurrencyMode.STATIC + + +def test_profile_for_maps_modes() -> None: + assert profile_for(ConcurrencyMode.STATIC) is None + assert profile_for(ConcurrencyMode.ADAPTIVE_CONSERVATIVE) is CONSERVATIVE + assert profile_for(ConcurrencyMode.ADAPTIVE_AGGRESSIVE) is AGGRESSIVE + assert CONSERVATIVE.step(100) == 1 + assert AGGRESSIVE.step(100) == 3 # 1 + isqrt(100)//4 + + +# --- pure control law ------------------------------------------------------ + + +def test_first_tick_probes_upward() -> None: + out = decide(CONSERVATIVE, _state(10, ewma=None), _signals(100), MIN, MAX) + assert out.permits == 11 + assert out.ewma_tput == 100 + assert out.direction == 1 + + +def test_clear_throughput_gain_climbs_in_direction() -> None: + out = decide(CONSERVATIVE, _state(10, ewma=100, direction=1), _signals(200), MIN, MAX) + assert out.permits == 11 # new_ewma 130 > band -> +1 + + +def test_clear_throughput_loss_reverses_direction() -> None: + out = decide(CONSERVATIVE, _state(10, ewma=100, direction=1), _signals(0), MIN, MAX) + assert out.direction == -1 + assert out.permits == 9 + + +def test_inside_dead_band_holds() -> None: + out = decide(CONSERVATIVE, _state(10, ewma=100, direction=1), _signals(101), MIN, MAX) + assert out.permits == 10 + + +@pytest.mark.parametrize("danger", ["gpu_temp_c", "cpu_pct", "ram_free_frac"]) +def test_critical_signal_forces_backoff_and_cooldown(danger: str) -> None: + crit = {"gpu_temp_c": 90.0, "cpu_pct": 98.0, "ram_free_frac": 0.05}[danger] + out = decide(CONSERVATIVE, _state(10, ewma=100), _signals(500, **{danger: crit}), MIN, MAX) + assert out.permits == 5 # halved + assert out.cool_down == CONSERVATIVE.cool_down_intervals + + +@pytest.mark.parametrize( + ("field", "value"), + [("cpu_pct", 92.0), ("ram_free_frac", 0.15), ("gpu_temp_c", 82.0), ("gpu_util_pct", 98.0)], +) +def test_soft_pressure_vetoes_increase(field: str, value: float) -> None: + # Throughput is rising, so without the veto it would climb; the veto holds it. + out = decide( + CONSERVATIVE, _state(10, ewma=100, direction=1), _signals(500, **{field: value}), MIN, MAX + ) + assert out.permits == 10 + + +def test_saturated_and_falling_steps_back() -> None: + # GPU saturated AND throughput slipping -> USL retrograde single step down. + out = decide( + CONSERVATIVE, _state(10, ewma=100, direction=1), _signals(0, gpu_util_pct=99.0), MIN, MAX + ) + assert out.permits == 9 + assert out.direction == -1 + + +def test_latency_gradient_vetoes_increase() -> None: + # An established low baseline (w_min) plus an inflated current residence time + # (permits high, throughput low) vetoes the climb before throughput rolls over. + out = decide(CONSERVATIVE, _state(20, ewma=100, direction=1, w_min=0.1), _signals(10), MIN, MAX) + assert out.permits == 20 # held: w_est 20/73 ~ 0.27 > 0.1 * 1.5 + + +def test_residence_baseline_is_tracked() -> None: + out = decide(CONSERVATIVE, _state(10, ewma=100, w_min=None), _signals(100), MIN, MAX) + assert out.w_min == pytest.approx(0.1) # 10 / 100 + + +def test_cooldown_suppresses_increase_and_decrements() -> None: + out = decide( + CONSERVATIVE, _state(10, ewma=100, direction=1, cool_down=2), _signals(500), MIN, MAX + ) + assert out.permits == 10 + assert out.cool_down == 1 + + +def test_clamped_to_max() -> None: + out = decide(CONSERVATIVE, _state(MAX, ewma=100, direction=1), _signals(500), MIN, MAX) + assert out.permits == MAX + + +def test_clamped_to_min_on_backoff() -> None: + out = decide(CONSERVATIVE, _state(1, ewma=100), _signals(500, cpu_pct=99.0), MIN, MAX) + assert out.permits == MIN + + +# --- resizable gate -------------------------------------------------------- + + +async def test_gate_blocks_at_limit_and_release_admits() -> None: + gate = ResizableGate(1) + await gate.acquire() + admitted = asyncio.Event() + + async def second() -> None: + await gate.acquire() + admitted.set() + + task = asyncio.create_task(second()) + await asyncio.sleep(0.01) + assert not admitted.is_set() + await gate.release() + await asyncio.wait_for(admitted.wait(), 1) + await task + + +async def test_gate_grow_wakes_a_waiter() -> None: + gate = ResizableGate(1) + await gate.acquire() + got = asyncio.Event() + + async def waiter() -> None: + await gate.acquire() + got.set() + + task = asyncio.create_task(waiter()) + await asyncio.sleep(0.01) + assert not got.is_set() + await gate.set_limit(2) + await asyncio.wait_for(got.wait(), 1) + await task + + +async def test_gate_shrink_lowers_ceiling() -> None: + gate = ResizableGate(2) + await gate.acquire() + await gate.acquire() # active == limit == 2 + await gate.set_limit(1) # over the new ceiling; must drain to below it + blocked = asyncio.Event() + + async def waiter() -> None: + await gate.acquire() + blocked.set() + + task = asyncio.create_task(waiter()) + await gate.release() # active 2 -> 1, still not < limit(1) + await asyncio.sleep(0.01) + assert not blocked.is_set() + await gate.release() # active 1 -> 0 < 1 + await asyncio.wait_for(blocked.wait(), 1) + await task + + +# --- controller loop ------------------------------------------------------- + + +async def test_controller_climbs_on_rising_throughput() -> None: + gate = ResizableGate(10) + done = {"c": 0} + tick = {"n": 0} + + def completed() -> int: + return done["c"] + + def sample(throughput: float) -> Signals: + return _signals(throughput) + + async def fake_sleep(_: float) -> None: + tick["n"] += 1 + done["c"] += 100 * tick["n"] # rising per-interval deltas + if tick["n"] >= 6: + raise asyncio.CancelledError + + ctrl = AdaptiveController( + gate, CONSERVATIVE, sample, completed, permit_min=1, permit_max=50, sleep=fake_sleep + ) + with contextlib.suppress(asyncio.CancelledError): + await ctrl.run() + assert gate.limit > 10 + + +async def test_controller_survives_a_sampler_failure() -> None: + gate = ResizableGate(10) + tick = {"n": 0} + + def completed() -> int: + return 0 + + def sample(throughput: float) -> Signals: + raise RuntimeError("probe blew up") + + async def fake_sleep(_: float) -> None: + tick["n"] += 1 + if tick["n"] >= 3: + raise asyncio.CancelledError + + ctrl = AdaptiveController( + gate, CONSERVATIVE, sample, completed, permit_min=1, permit_max=50, sleep=fake_sleep + ) + with contextlib.suppress(asyncio.CancelledError): + await ctrl.run() + assert gate.limit == 10 # tick failures swallowed; the tuner never crashed + + +async def test_controller_holds_and_skips_resize_when_flat() -> None: + gate = ResizableGate(10) + done = {"c": 0} + tick = {"n": 0} + + def completed() -> int: + return done["c"] + + def sample(throughput: float) -> Signals: + return _signals(throughput) + + async def fake_sleep(_: float) -> None: + tick["n"] += 1 + done["c"] += 100 # constant delta -> flat throughput after the first probe + if tick["n"] >= 4: + raise asyncio.CancelledError + + ctrl = AdaptiveController( + gate, CONSERVATIVE, sample, completed, permit_min=1, permit_max=50, sleep=fake_sleep + ) + with contextlib.suppress(asyncio.CancelledError): + await ctrl.run() + assert gate.limit == 11 # one probe up, then flat -> holds (no further resize) + + +async def test_controller_backs_off_on_emergency() -> None: + gate = ResizableGate(20) + tick = {"n": 0} + + def completed() -> int: + return 0 + + def sample(throughput: float) -> Signals: + return _signals(throughput, gpu_temp_c=90.0) # over TEMP_CRIT + + async def fake_sleep(_: float) -> None: + tick["n"] += 1 + if tick["n"] >= 2: # let one decide() run, then stop + raise asyncio.CancelledError + + ctrl = AdaptiveController( + gate, CONSERVATIVE, sample, completed, permit_min=1, permit_max=50, sleep=fake_sleep + ) + with contextlib.suppress(asyncio.CancelledError): + await ctrl.run() + assert gate.limit < 20 + + +async def test_gate_context_manager_admits_and_releases() -> None: + gate = ResizableGate(1) + async with gate: + blocked = asyncio.Event() + + async def waiter() -> None: + async with gate: + blocked.set() + + task = asyncio.create_task(waiter()) + await asyncio.sleep(0.01) + assert not blocked.is_set() # held inside the outer `async with` + await asyncio.wait_for(blocked.wait(), 1) # released on exit + await task + + +# --- real signal sampler --------------------------------------------------- + + +def test_sampler_reports_mean_util_and_max_temp(monkeypatch: pytest.MonkeyPatch) -> None: + from lilbee.providers.fleet import gpu_stats + + stats = { + 0: gpu_stats.GpuStat(0, utilization_pct=40, free_bytes=1, total_bytes=2, temperature_c=70), + 1: gpu_stats.GpuStat(1, utilization_pct=80, free_bytes=1, total_bytes=2, temperature_c=85), + } + monkeypatch.setattr(gpu_stats, "probe_gpu_stats", lambda _devices: stats) + signals = make_signal_sampler([object()])(throughput=12.0) + assert signals.throughput == 12.0 + assert signals.gpu_util_pct == 60.0 # mean(40, 80) + assert signals.gpu_temp_c == 85.0 # max(70, 85) + assert 0.0 <= signals.ram_free_frac <= 1.0 + + +def test_sampler_without_gpu_telemetry_is_none(monkeypatch: pytest.MonkeyPatch) -> None: + from lilbee.providers.fleet import gpu_stats + + monkeypatch.setattr(gpu_stats, "probe_gpu_stats", lambda _devices: {}) + signals = make_signal_sampler([])(throughput=0.0) + assert signals.gpu_util_pct is None + assert signals.gpu_temp_c is None + + +# --- device enumeration ---------------------------------------------------- + + +def test_enumerate_devices_returns_probe_result(monkeypatch: pytest.MonkeyPatch) -> None: + from lilbee.providers.fleet import binary, planning + + sentinel = [object()] + monkeypatch.setattr(binary, "resolve_llama_server", lambda: "llama-server") + monkeypatch.setattr(planning, "resolve_devices", lambda _binary: sentinel) + assert enumerate_fleet_devices() == sentinel + + +def test_enumerate_devices_degrades_to_empty(monkeypatch: pytest.MonkeyPatch) -> None: + from lilbee.providers.fleet import binary + + def boom() -> str: + raise RuntimeError("no engine") + + monkeypatch.setattr(binary, "resolve_llama_server", boom) + assert enumerate_fleet_devices() == [] From ca1d5744f26c19aa8d4ed7e1a15662a65e4782fb Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:00:17 -0400 Subject: [PATCH 44/77] fix(ingest): surface silently dropped document files after a sync A document that discovery found but that never made it into the index -- and was never reported as failed or skipped -- vanished with no signal, so a whole dataset could be missing from search with the sync reporting success. After each sync, reconcile the on-disk document files against the sources table and log a warning naming any that ended up in neither the index nor the failed/skipped lists, so a silent drop is loud instead. Discovery, extraction, rasterization, the store round-trip, and skip markers all handle filenames with spaces correctly (verified end-to-end), so this adds the reconciliation guard and a spaced-filename regression test rather than a change to a path that already works. (cherry picked from commit f86573c7c8c4e2cf3f2cb1cab83314adc1720c70) --- src/lilbee/data/ingest/pipeline.py | 31 ++++++++++++++++++++ tests/test_ingest.py | 45 ++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/src/lilbee/data/ingest/pipeline.py b/src/lilbee/data/ingest/pipeline.py index fd274f890..68b56c885 100644 --- a/src/lilbee/data/ingest/pipeline.py +++ b/src/lilbee/data/ingest/pipeline.py @@ -379,6 +379,25 @@ def _force_rebuild_store(store: Any) -> None: store.rebuild_memory_embeddings(lambda texts: embedder.embed_batch(texts)) +def _reconcile_missing( + disk_files: dict[str, Path], + sources: list[SourceRecord], + failed: Iterable[str], + skipped: Iterable[str], +) -> list[str]: + """On-disk document files absent from the store and not reported failed/skipped. + + A file discovery found and classified that ended up in neither the sources table + nor the run's failed/skipped lists was dropped with no signal -- the silent + data-loss case (a scanned PDF that never made it into the index yet reported no + error). Everything legitimately not indexed is excluded: a failed extraction is in + ``failed``, a zero-text file is in ``skipped``, and an unsupported type was never + returned by discovery in the first place. + """ + accounted = {s["filename"] for s in sources} | set(failed) | set(skipped) + return sorted(name for name in disk_files if name not in accounted) + + async def sync( force_rebuild: bool = False, quiet: bool = False, @@ -474,6 +493,18 @@ async def sync( await incremental_update(set(added) | set(updated) | set(removed)) + # Reconciliation guard against silent data loss: any on-disk document file that + # ended up in neither the index nor the failed/skipped lists was dropped without + # a signal. Surface it loudly instead of letting a whole dataset vanish quietly. + missing = _reconcile_missing(disk_files, _store.get_sources(), failed, skipped) + if missing: + log.warning( + "Sync reconciliation: %d document file(s) on disk are absent from the index " + "with no failure reported (possible silent drop): %s", + len(missing), + ", ".join(missing[:20]), + ) + result = SyncResult( added=list(added), updated=list(updated), diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 0b9bec49c..a3b33b9ca 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -176,6 +176,17 @@ def _make_empty_result(): return result +def test_reconcile_missing_flags_only_silent_drops(): + from pathlib import Path + + from lilbee.data.ingest.pipeline import _reconcile_missing + + disk = {n: Path(n) for n in ("a.pdf", "b.pdf", "c.pdf", "d.pdf")} + # a indexed, b failed, c skipped, d dropped with no signal. + missing = _reconcile_missing(disk, [{"filename": "a.pdf"}], failed=["b.pdf"], skipped=["c.pdf"]) + assert missing == ["d.pdf"] + + @mock.patch( "lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock, @@ -215,6 +226,40 @@ async def test_adaptive_mode_ingests_and_stops_controller( result = await sync(quiet=True) assert "adaptive.txt" in result.added + async def test_spaced_filename_ingests_without_silent_drop( + self, mock_extract_file, isolated_env, caplog + ): + # Regression for the silent drop of files with spaces in their names. + (isolated_env / "Request No. 1.txt").write_text("Maintenance request approved, page one.") + import logging + + from lilbee.data.ingest import sync + + with caplog.at_level(logging.WARNING, logger="lilbee.data.ingest.pipeline"): + result = await sync(quiet=True) + assert "Request No. 1.txt" in result.added + assert not any("reconciliation" in r.getMessage().lower() for r in caplog.records) + + async def test_reconciliation_warns_on_silent_drop( + self, mock_extract_file, isolated_env, monkeypatch, caplog + ): + # A file discovered but never indexed, failed, or skipped is a silent drop. + (isolated_env / "ghost.txt").write_text("This file gets dropped by a broken ingest stage.") + import logging + + from lilbee.data.ingest import pipeline, sync + + async def _noop_ingest(*_a, **_k): + return None + + monkeypatch.setattr(pipeline, "ingest_batch", _noop_ingest) + with caplog.at_level(logging.WARNING, logger="lilbee.data.ingest.pipeline"): + await sync(quiet=True) + assert any( + "reconciliation" in r.getMessage().lower() and "ghost.txt" in r.getMessage() + for r in caplog.records + ) + async def test_quiet_mode_suppresses_progress(self, mock_extract_file, isolated_env): (isolated_env / "quiet.txt").write_text("Quiet mode test content.") from lilbee.data.ingest import sync From 2605a25ac311bf03f549b28307c99719c4c64640 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Thu, 9 Jul 2026 00:39:59 -0400 Subject: [PATCH 45/77] fix(ingest): cap the extraction pool default at 32 (OCR ingest is GPU-bound) A forced-OCR worker-count sweep on multi-page PDFs (4x H100) showed ingest throughput flat from 16 to 32 workers and declining past it, with the GPUs ~85-90% busy the whole time: OCR is GPU-bound, not extraction-bound. Scaling the pool to the vCPU count (cpu+4, ~212 on a big box) only oversubscribes without feeding the GPUs faster and mildly regresses throughput, so restore the min(32, cpu+4) default. The LILBEE_INGEST_MAX_WORKERS override still lifts the cap for genuinely CPU-bound bulk extraction. Refs bb-1keb. --- src/lilbee/data/ingest/offload.py | 19 +++++++++++-------- tests/test_ingest_offload.py | 13 ++++++++----- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/lilbee/data/ingest/offload.py b/src/lilbee/data/ingest/offload.py index 977d31050..b4bd1fb3e 100644 --- a/src/lilbee/data/ingest/offload.py +++ b/src/lilbee/data/ingest/offload.py @@ -28,13 +28,16 @@ def _max_workers() -> int: """Concurrent slots for ingest offload work; honors ``LILBEE_INGEST_MAX_WORKERS``. - Extraction rasterizes PDFs and drives OCR on this pool, so its width caps how - many documents feed the GPU OCR/embed slots at once. The old fixed ``32`` - ceiling left most cores idle on a many-vCPU box and pinned full-corpus - throughput below the vision fleet's capacity, so the default now scales with - the vCPU count (``os.cpu_count() + 4``, the ``+4`` headroom for threads - parked on OCR/embed I/O). The override accepts a positive integer; non-positive - or unparseable values fall back to the default and a warning is logged. + Extraction rasterizes PDFs and drives OCR on this pool. The default caps at + ``min(32, cpu_count + 4)``: a worker-count sweep on forced-OCR multi-page PDFs + (4x H100) held throughput flat from 16 to 32 workers and *declining* past it, + with the GPUs already ~85-90% busy the whole time. OCR ingest is GPU-bound, not + extraction-bound, so extra threads only rasterize ahead into a buffer the GPUs + cannot drain any faster while oversubscribing the box. The ``+4`` keeps headroom + for threads parked on OCR/embed I/O; small hosts scale below the cap. The + override lifts the ceiling for genuinely CPU-bound work (e.g. bulk text + extraction) that can use more threads; non-positive or unparseable values warn + and fall back to the default. """ override = os.environ.get(_MAX_WORKERS_ENV) if override is not None: @@ -49,7 +52,7 @@ def _max_workers() -> int: _MAX_WORKERS_ENV, override, ) - return (os.cpu_count() or 4) + 4 + return min(32, (os.cpu_count() or 4) + 4) def max_workers() -> int: diff --git a/tests/test_ingest_offload.py b/tests/test_ingest_offload.py index 819b0ca77..503a1a6c9 100644 --- a/tests/test_ingest_offload.py +++ b/tests/test_ingest_offload.py @@ -2,8 +2,9 @@ Extraction (PDF rasterization + OCR) runs on this pool, so its width caps how many documents rasterize in parallel to feed the GPU OCR/embed slots on a -full-corpus run. These lock in that the width scales with the vCPU count and is -operator-overridable, rather than the old fixed 32-thread ceiling. +full-corpus run. OCR ingest is GPU-bound, so these lock in a default capped at +``min(32, cpu_count + 4)`` (a worker sweep showed throughput flat/declining past +~32) that stays operator-overridable via ``LILBEE_INGEST_MAX_WORKERS``. """ from __future__ import annotations @@ -18,9 +19,11 @@ def _clear_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("LILBEE_INGEST_MAX_WORKERS", raising=False) -def test_default_scales_past_the_old_32_ceiling(monkeypatch: pytest.MonkeyPatch) -> None: +def test_default_caps_at_32_on_a_big_box(monkeypatch: pytest.MonkeyPatch) -> None: + # OCR ingest is GPU-bound; a worker sweep showed throughput flat/declining past + # ~32, so the default caps there instead of scaling to the vCPU count. monkeypatch.setattr(offload.os, "cpu_count", lambda: 128) - assert offload._max_workers() == 132 + assert offload._max_workers() == 32 def test_default_keeps_headroom_on_a_small_box(monkeypatch: pytest.MonkeyPatch) -> None: @@ -33,7 +36,7 @@ def test_default_falls_back_when_cpu_count_is_unknown(monkeypatch: pytest.Monkey assert offload._max_workers() == 8 -def test_env_override_wins_over_the_scaled_default(monkeypatch: pytest.MonkeyPatch) -> None: +def test_env_override_wins_over_the_capped_default(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(offload.os, "cpu_count", lambda: 8) monkeypatch.setenv("LILBEE_INGEST_MAX_WORKERS", "200") assert offload._max_workers() == 200 From 0d2d2708e1c600a608d09952ffee2df9af168c11 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:21:59 -0400 Subject: [PATCH 46/77] style: format test_ingest.py --- tests/test_ingest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ingest.py b/tests/test_ingest.py index a3b33b9ca..0f2491d8f 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -221,7 +221,7 @@ async def test_adaptive_mode_ingests_and_stops_controller( monkeypatch.setattr( pipeline, "make_signal_sampler", - lambda _devices: (lambda t: Signals(t, 50.0, 60.0, 50.0, 0.5)), + lambda _devices: lambda t: Signals(t, 50.0, 60.0, 50.0, 0.5), ) result = await sync(quiet=True) assert "adaptive.txt" in result.added From 99092c12a37257806abd86d06811ef6bc7db47ea Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:21:59 -0400 Subject: [PATCH 47/77] chore: step up to xberg 1.0.0rc22 from PyPI The local rc9 trial-wheel source override is gone: xberg now resolves from the index like every other dependency. The Intel Mac release cell drops its compat-index xberg pin since rc22 ships an x86_64 macOS wheel on PyPI, and comments that dated themselves to rc9 are reworded. --- .github/workflows/release.yml | 19 ++++++++----------- pyproject.toml | 11 +---------- src/lilbee/data/ingest/vision_ocr_backend.py | 4 ++-- uv.lock | 13 +++++++++---- 4 files changed, 20 insertions(+), 27 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c0dacc31f..147c3a224 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -239,16 +239,13 @@ jobs: if: matrix.os != 'macos-15-intel' run: uv sync --extra release && uv pip install 'nuitka[onefile]>=4.0,<5' pip - # Intel Mac: `uv sync` can't run here for two reasons -- stock lancedb 0.33.0 - # ships no macosx_x86_64 wheel, and xberg is pinned to a local arm64 dev wheel - # via [tool.uv.sources]. Resolve both from the +compat index (PEP 440 drop-ins), - # then install the rest of [release] fresh from PyPI (every other dep has an - # x86_64 macOS wheel). CMAKE_ARGS + deployment target hold any throwaway - # llama-cpp-python sdist build at the cpu/x86_64/macOS-11 baseline; the dedicated - # step below builds the llama-cpp-python that actually ships. - # NOTE: gated on xberg landing an x86_64 macOS wheel on the +compat index; until - # then this cell is expected to fail at install (it is soft, so it never blocks - # the release fan-out). + # Intel Mac: `uv sync` can't run here -- stock lancedb 0.33.0 ships no + # macosx_x86_64 wheel. Resolve it from the +compat index (PEP 440 drop-in), + # then install the rest of [release] fresh from PyPI (xberg >=1.0.0rc22 and + # every other dep ship an x86_64 macOS wheel). CMAKE_ARGS + deployment target + # hold any throwaway llama-cpp-python sdist build at the cpu/x86_64/macOS-11 + # baseline; the dedicated step below builds the llama-cpp-python that actually + # ships. The cell stays soft so an install failure never blocks the fan-out. - name: Install dependencies (Intel Mac) if: matrix.os == 'macos-15-intel' shell: bash @@ -260,7 +257,7 @@ jobs: export CMAKE_ARGS uv venv uv pip install --extra-index-url https://lilbee.sh/compat/ \ - lancedb==0.33.0+compat xberg==1.0.0rc5 + lancedb==0.33.0+compat uv pip install '.[release]' 'nuitka[onefile]>=4.0,<5' pip - name: Verify extras installed in build venv diff --git a/pyproject.toml b/pyproject.toml index 24f4c62e9..fbff71ae5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ classifiers = [ dependencies = [ "lancedb", - "xberg>=1.0.0rc1", + "xberg>=1.0.0rc22", "filelock", "tree-sitter-language-pack>=1.8.0,<2.0", "typer>=0.12", @@ -98,15 +98,6 @@ packages = ["src/lilbee"] # the index instead -- this source override is workspace metadata, not published deps. [tool.uv.sources] lilbee-engine = { path = "packaging/engine-wheel" } -# TODO: pre-publication trial wheel; resolve it from the local build until it -# lands on PyPI, then drop this source and lock from the index. Built from xberg -# 1.0.0rc9 built from kreuzberg main (293f1eaadc), bindings regenerated with -# alef 0.32.6. Carries the OcrBackend host-method forwarding fix (kreuzberg-3u2), -# the public options.OcrConfig Protocol (kreuzberg-u4r), and the merged -# tokenizer-backend sizing feature (PR #1196 / kreuzberg-1h6): ChunkSizing -# resolves a registered TokenizerBackend name registry-first, so lilbee can size -# chunks with its own embedder tokenizer instead of the char-budget heuristic. -xberg = { path = "/Users/tobias/projects/kreuzberg/dist/xberg-1.0.0rc9-cp310-abi3-macosx_11_0_arm64.whl" } [dependency-groups] dev = [ diff --git a/src/lilbee/data/ingest/vision_ocr_backend.py b/src/lilbee/data/ingest/vision_ocr_backend.py index 90f380f25..e849c7275 100644 --- a/src/lilbee/data/ingest/vision_ocr_backend.py +++ b/src/lilbee/data/ingest/vision_ocr_backend.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator - # rc9's OcrBackend Protocol types the callback config as the public + # The OcrBackend Protocol types the callback config as the public # xberg.OcrConfig (kreuzberg-u4r); a backend typed against the native # xberg._xberg.OcrConfig no longer satisfies it. The runtime object still # arrives with backend_options as a JSON string (handled in _OcrConfigView). @@ -84,7 +84,7 @@ def backend_options_for(token: str) -> dict[str, str]: class _OcrConfigView: """Typed reader over the xberg OcrConfig object passed to process_image. - Typed against the public ``xberg.OcrConfig`` (rc9's OcrBackend Protocol), whose + Typed against the public ``xberg.OcrConfig`` from the OcrBackend Protocol, whose fields are read directly as attributes. The native round-trip hands ``backend_options`` back as a JSON string rather than the dict lilbee put in, so ``request_token`` accepts both shapes. diff --git a/uv.lock b/uv.lock index fd7de5c87..78e099a68 100644 --- a/uv.lock +++ b/uv.lock @@ -1710,7 +1710,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "typing-extensions", specifier = ">=4.5.0" }, { name = "uvicorn", specifier = ">=0.30" }, - { name = "xberg", path = "../../../Users/tobias/projects/kreuzberg/dist/xberg-1.0.0rc9-cp310-abi3-macosx_11_0_arm64.whl" }, + { name = "xberg", specifier = ">=1.0.0rc22" }, ] provides-extras = ["litellm", "graph", "crawler", "cuda12", "release"] @@ -4452,10 +4452,15 @@ wheels = [ [[package]] name = "xberg" -version = "1.0.0rc9" -source = { path = "../../../Users/tobias/projects/kreuzberg/dist/xberg-1.0.0rc9-cp310-abi3-macosx_11_0_arm64.whl" } +version = "1.0.0rc22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/71/2d/8abe3545eb46529b2ad40c3d5cb568b37c1a0e023a87d01355a11b42b879/xberg-1.0.0rc22.tar.gz", hash = "sha256:3c6394ac5c0efe9512a6eca97376c7db00211ab6cd898f2f78dd5517e61cd8b0", size = 301005 } wheels = [ - { filename = "xberg-1.0.0rc9-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:86b4248c863f69a9fe575f49ceaddeb395882115c11ef1598cd2ed49748f9a7c" }, + { url = "https://files.pythonhosted.org/packages/12/c1/c106d45c49a89d3942a6b7e83f538a17f63124ca6649589d66eddd6dce7f/xberg-1.0.0rc22-cp310-abi3-macosx_15_0_arm64.whl", hash = "sha256:ef07ef2908662577eee35731743529f8bd763066b040f7afec073d751da9c356", size = 40207943 }, + { url = "https://files.pythonhosted.org/packages/0d/33/8ec6e629a83606e15ec6e0caec53b6008b9f7b036fbe3b2f8c41237af581/xberg-1.0.0rc22-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:6ffa34cf46fdac90a5130944e77cae95f3f6d96f7968e329bec997a5e7a95f38", size = 37567655 }, + { url = "https://files.pythonhosted.org/packages/48/6b/796ac3589447f1c44981bbeb0b89ce6dd56a93b29a4ec99d391e9370efc4/xberg-1.0.0rc22-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ef94594e12d0aac8dc5e789de7996ae4452323ccbbbc52db2a9753d26302eff9", size = 41765830 }, + { url = "https://files.pythonhosted.org/packages/9d/f5/065746a4401250013a255226a912832d887228be72a58854cc53a09dd357/xberg-1.0.0rc22-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e3d1e3c8ebca420dc472584d27cd96e661f36b364e8bc9b8c357e29189d8c361", size = 44169220 }, + { url = "https://files.pythonhosted.org/packages/db/b3/a2902f0a72b1a0e6797ecd34882ed775f2ffc8b6805f40ae9a3a28c44f7c/xberg-1.0.0rc22-cp310-abi3-win_amd64.whl", hash = "sha256:963c1870d987792ebdc1f8135bfa9506b430fffa9d66a8e46074d72d9dacc576", size = 30692469 }, ] [[package]] From 7401868f3ea03fb91209e74f9e14bb49cb588a35 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:48:12 -0400 Subject: [PATCH 48/77] Drop the accidentally-committed WIP progress test again The untracked xet-progress WIP file got swept into the merge commit by a broad git add during conflict resolution, repeating what c48a36d7 removed. It downloads a real model from the network and uses the POSIX-only os.killpg, so it fails CI on transient CDN truncation and on Windows. --- .../test_xet_progress_granularity.py | 297 ------------------ 1 file changed, 297 deletions(-) delete mode 100644 tests/integration/test_xet_progress_granularity.py diff --git a/tests/integration/test_xet_progress_granularity.py b/tests/integration/test_xet_progress_granularity.py deleted file mode 100644 index 4d516f3c0..000000000 --- a/tests/integration/test_xet_progress_granularity.py +++ /dev/null @@ -1,297 +0,0 @@ -"""Bounded-prefix QA matrix: real-time download-progress granularity, xet vs HTTP. - -WHY: lilbee sets ``HF_HUB_DISABLE_XET=1`` (src/lilbee/__init__.py) because the xet -transfer layer historically reported download progress in 3-4 coarse jumps (HF issue -#4058), making bars look stuck on large files. Forcing the HTTP path restores smooth -per-chunk updates. This harness empirically measures whether hf-xet's transfer layer -now drives our progress callback (catalog/download_progress.py) at chunk granularity, -i.e. whether the bypass can be dropped. - -It NEVER downloads the whole file. Each transfer path runs in a fresh subprocess -(``HF_HUB_DISABLE_XET`` is read at import time, so it must be set before ``import -lilbee``). The child streams progress samples; the parent kills it once a bounded byte -prefix has been sampled. Total bandwidth per run is ~2x ``PREFIX_BUDGET_BYTES``. - -Manual run (prints the matrix + verdict): - uv run python tests/integration/test_xet_progress_granularity.py -Pytest (slow): - uv run pytest tests/integration/test_xet_progress_granularity.py -v -m slow -""" - -from __future__ import annotations - -import contextlib -import os -import signal -import statistics -import subprocess -import sys -import tempfile -import time -from dataclasses import dataclass, field -from pathlib import Path - -import pytest - -pytestmark = pytest.mark.slow - -# A xet-backed GGUF (get_hf_file_metadata reports XetFileData for this file). -# ~1.0 GB total; we only sample the prefix, so the full size never downloads. -TEST_REPO = "unsloth/Qwen3-4B-GGUF" -TEST_FILE = "Qwen3-4B-UD-IQ1_S.gguf" - -# Stop each download once this many bytes have been observed. Large enough to -# reveal cadence on the xet path, small enough to stay cheap. -PREFIX_BUDGET_BYTES = 400 * 1024 * 1024 -# Hard wall-clock cap per path, so a stalled transfer can't hang the run. -WALL_TIMEOUT_S = 600.0 - -# Thresholds for "real-time" over the sampled prefix. The HTTP path (200KB chunks, -# see _shrink_hf_download_chunk_size) is the known-good baseline and easily clears -# these; the xet path is what we're judging. -MIN_UPDATES = 20 -MAX_JUMP_PCT = 10.0 - -_SAMPLE_PREFIX = "S\t" - - -@dataclass -class Metrics: - path: str - samples: list[tuple[int, int, float]] = field(default_factory=list) # (bytes, total, ts) - error: str | None = None - - @property - def n_updates(self) -> int: - """Distinct forward progress updates (a sample whose byte count grew).""" - count = 0 - prev = -1 - for done, _total, _ts in self.samples: - if done > prev: - count += 1 - prev = done - return count - - @property - def total_size(self) -> int: - for _done, total, _ts in self.samples: - if total > 0: - return total - return 0 - - @property - def max_jump_pct(self) -> float: - """Largest single-update byte jump as a percent of the full file size.""" - total = self.total_size - if total <= 0 or len(self.samples) < 2: - return 100.0 - prev = self.samples[0][0] - worst = 0 - for done, _total, _ts in self.samples[1:]: - worst = max(worst, done - prev) - prev = done - return worst * 100.0 / total - - @property - def gaps(self) -> list[float]: - return [b[2] - a[2] for a, b in zip(self.samples, self.samples[1:], strict=False)] - - @property - def max_gap_s(self) -> float: - gaps = self.gaps - return max(gaps) if gaps else 0.0 - - @property - def median_gap_s(self) -> float: - gaps = self.gaps - return statistics.median(gaps) if gaps else 0.0 - - @property - def monotonic(self) -> bool: - prev = -1 - for done, _total, _ts in self.samples: - if done < prev: - return False - prev = done - return True - - @property - def is_smooth(self) -> bool: - return ( - self.error is None - and self.n_updates >= MIN_UPDATES - and self.max_jump_pct <= MAX_JUMP_PCT - and self.monotonic - ) - - -def _child_download(models_dir: str) -> int: - """Child entry point: download into ``models_dir``, stream progress, abort at budget. - - Runs in a fresh process so the parent's ``HF_HUB_DISABLE_XET`` env is honored. - """ - import lilbee # noqa: F401 # sets HF env from the inherited environment - from lilbee.catalog import CatalogModel, download_model - from lilbee.core.config.model import cfg - from lilbee.runtime.cancellation import TaskCancelledError - - cfg.models_dir = Path(models_dir) - cfg.models_dir.mkdir(parents=True, exist_ok=True) - - entry = CatalogModel( - hf_repo=TEST_REPO, - gguf_filename=TEST_FILE, - size_gb=1.0, - min_ram_gb=2.0, - description="xet granularity probe", - featured=False, - downloads=0, - task="chat", - ) - - def on_progress(downloaded: int, total: int) -> None: - sys.stdout.write(f"{_SAMPLE_PREFIX}{downloaded}\t{total}\t{time.monotonic()}\n") - sys.stdout.flush() - if downloaded >= PREFIX_BUDGET_BYTES: - # Best-effort clean stop (download.py re-raises TaskCancelledError). - # The parent also hard-kills at budget in case xet swallows this. - raise TaskCancelledError - - try: - download_model(entry, on_progress=on_progress) - except TaskCancelledError: - pass - except Exception as exc: - sys.stdout.write(f"ERR\t{type(exc).__name__}: {exc}\n") - sys.stdout.flush() - return 1 - return 0 - - -def _measure(path: str, base_dir: Path) -> Metrics: - """Run one transfer path in a subprocess, killing it once the prefix is sampled.""" - assert path in ("xet", "http") - models_dir = base_dir / "models" - hf_home = base_dir / "hf_home" - models_dir.mkdir(parents=True, exist_ok=True) - hf_home.mkdir(parents=True, exist_ok=True) - - env = os.environ.copy() - env["HF_HUB_DISABLE_XET"] = "0" if path == "xet" else "1" - env["HF_HOME"] = str(hf_home) # isolates the xet chunk-cache for a cold download - env["HF_HUB_DISABLE_PROGRESS_BARS"] = "1" - - proc = subprocess.Popen( - [sys.executable, __file__, "--child", str(models_dir)], - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - bufsize=1, - start_new_session=True, - ) - metrics = Metrics(path=path) - deadline = time.monotonic() + WALL_TIMEOUT_S - try: - assert proc.stdout is not None - for line in proc.stdout: - if time.monotonic() > deadline: - metrics.error = metrics.error or "wall-clock timeout" - break - if line.startswith("ERR\t"): - metrics.error = line[4:].strip() - break - if not line.startswith(_SAMPLE_PREFIX): - continue - try: - done_s, total_s, ts_s = line[len(_SAMPLE_PREFIX) :].split("\t") - metrics.samples.append((int(done_s), int(total_s), float(ts_s))) - except ValueError: - continue - if metrics.samples[-1][0] >= PREFIX_BUDGET_BYTES: - break - finally: - _kill_group(proc) - return metrics - - -def _kill_group(proc: subprocess.Popen[str]) -> None: - if proc.poll() is None: - try: - os.killpg(os.getpgid(proc.pid), signal.SIGKILL) - except (ProcessLookupError, PermissionError): - proc.kill() - with contextlib.suppress(subprocess.TimeoutExpired): - proc.wait(timeout=10) - - -def run_matrix(base_dir: Path) -> tuple[Metrics, Metrics]: - """Measure both transfer paths under isolated, cold caches.""" - http = _measure("http", base_dir / "http") - xet = _measure("xet", base_dir / "xet") - return http, xet - - -def _row(m: Metrics) -> str: - if m.error and not m.samples: - return f" {m.path:<5} | ERROR: {m.error}" - verdict = "PASS" if m.is_smooth else "FAIL" - return ( - f" {m.path:<5} | {verdict} | updates={m.n_updates:>5} " - f"max_jump={m.max_jump_pct:6.2f}% max_gap={m.max_gap_s:6.2f}s " - f"median_gap={m.median_gap_s:6.3f}s monotonic={m.monotonic}" - ) - - -def format_matrix(http: Metrics, xet: Metrics) -> str: - total_mb = (http.total_size or xet.total_size) / (1024 * 1024) - lines = [ - "", - "Real-time download-progress QA matrix", - f" model: {TEST_REPO}/{TEST_FILE} (~{total_mb:.0f} MB total, " - f"sampled prefix ~{PREFIX_BUDGET_BYTES // (1024 * 1024)} MB)", - f" bar: >= {MIN_UPDATES} updates, max single jump <= {MAX_JUMP_PCT:.0f}% of file", - _row(http), - _row(xet), - ] - if xet.error is None and xet.samples: - if xet.is_smooth: - lines.append(" VERDICT: xet now delivers real-time progress.") - lines.append( - " -> RECOMMEND dropping HF_HUB_DISABLE_XET=1 (re-check the chunk shrink)." - ) - else: - lines.append(" VERDICT: xet progress is still coarse; keep the HTTP bypass.") - lines.append("") - return "\n".join(lines) - - -def test_progress_granularity_matrix(tmp_path: Path) -> None: - """Run the xet-vs-HTTP matrix; guard the HTTP baseline, report the xet verdict.""" - http, xet = run_matrix(tmp_path) - print(format_matrix(http, xet)) - - if http.error and not http.samples: - pytest.skip(f"HTTP baseline unavailable (network?): {http.error}") - - # The HTTP path is lilbee's shipped default: it must stay smooth (regression guard). - assert http.is_smooth, f"HTTP baseline regressed: {_row(http)}" - - # The xet path is under investigation and disabled by default. We only require - # that it produces progress at all; smoothness is reported, not asserted. - if xet.error and not xet.samples: - pytest.skip(f"xet path unavailable (network?): {xet.error}") - assert xet.n_updates >= 1, "xet path produced no progress samples" - - -def _main() -> int: - with tempfile.TemporaryDirectory(prefix="lilbee-xet-qa-") as tmp: - http, xet = run_matrix(Path(tmp)) - print(format_matrix(http, xet)) - return 0 - - -if __name__ == "__main__": - if len(sys.argv) >= 3 and sys.argv[1] == "--child": - sys.exit(_child_download(sys.argv[2])) - sys.exit(_main()) From 68e7331039b38ef428f7b035e9cfecd4835b0ac1 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:10:24 -0400 Subject: [PATCH 49/77] fix(services): serialize singleton creation and xberg backend re-binds Concurrent first-touch get_services() calls (e.g. several download workers) each built a full container, and the duplicate builds collided in xberg's process-global backend registry: the losing thread raised 'Embedding backend lilbee is already registered' and its task failed. Before the xberg migration a duplicate build was benign (last one won), so the old lock-free contract no longer holds. Singleton creation is now double-checked under a lock, and each backend re-bind (unregister + register) is atomic, so the most recently built provider wins instead of throwing. --- src/lilbee/app/services.py | 111 ++++++++++++++++++++++--------------- tests/test_services.py | 46 +++++++++++++++ 2 files changed, 111 insertions(+), 46 deletions(-) diff --git a/src/lilbee/app/services.py b/src/lilbee/app/services.py index d279837eb..1c46fd4ab 100644 --- a/src/lilbee/app/services.py +++ b/src/lilbee/app/services.py @@ -122,11 +122,11 @@ class _ServicesState: """The cached process-global singleton plus the per-task scoped override. ``singleton`` is set on first ``get_services()`` call. Concurrency - contract: lilbee runs the asyncio loop on a single worker thread + - Textual's main thread; ``get_services()`` is idempotent (the early-out - covers re-entry from a background thread), and the Services dataclass is - logically immutable post-construction, so concurrent reads are safe - without a lock. Tests that need a custom container call + contract: creation is serialized by ``_singleton_create_lock`` (several + worker threads can first-touch services at once, and a duplicate build + would collide in xberg's process-global backend registry), and the + Services dataclass is logically immutable post-construction, so concurrent + reads are safe without a lock. Tests that need a custom container call ``set_services(make_mock_services(...))``; ``peek_services()`` is the read-only inspector for cleanup fixtures. @@ -226,6 +226,18 @@ def build_services( ) +# Serializes first-touch singleton creation: several worker threads (e.g. +# concurrent downloads) can call get_services() before the singleton exists, +# and a duplicate build re-registers xberg's process-global backends mid-flight, +# raising 'already registered' in the losing thread. +_singleton_create_lock = threading.Lock() + +# Makes each xberg registry re-bind (unregister + register) atomic, so +# concurrent binds serialize and the most recently built provider wins instead +# of colliding on 'already registered'. +_backend_bind_lock = threading.Lock() + + def get_services() -> Services: """Return the active container: a scoped override if set, else the cached singleton. @@ -240,26 +252,30 @@ def get_services() -> Services: if _state.singleton is not None: return _state.singleton - from lilbee.app.settings import reconcile_embedding_dim - from lilbee.core.config import cfg - from lilbee.modelhub.registry import ModelRegistry - - registry = ModelRegistry(cfg.models_dir) - # Pin the store width to the embedder before Store(); pass the registry so - # resolution doesn't re-enter this half-built get_services. - reconcile_embedding_dim(registry) - _state.singleton = build_services(cfg, registry=registry) - # Eager start is the default: pay the spawn cost per role server at TUI mount - # so the first user action lands on a warm fleet. Roles whose model is unset - # are skipped, so a setup with only chat + embed never spawns rerank or - # vision. Set ``cfg.worker_pool_eager_start = false`` for headless scripts - # where mount time matters more than first-call latency. - if cfg.worker_pool_eager_start: - from contextlib import suppress - - with suppress(Exception): - _state.singleton.provider.warm_up_pool() - return _state.singleton + with _singleton_create_lock: + if _state.singleton is not None: + return _state.singleton + + from lilbee.app.settings import reconcile_embedding_dim + from lilbee.core.config import cfg + from lilbee.modelhub.registry import ModelRegistry + + registry = ModelRegistry(cfg.models_dir) + # Pin the store width to the embedder before Store(); pass the registry so + # resolution doesn't re-enter this half-built get_services. + reconcile_embedding_dim(registry) + _state.singleton = build_services(cfg, registry=registry) + # Eager start is the default: pay the spawn cost per role server at TUI mount + # so the first user action lands on a warm fleet. Roles whose model is unset + # are skipped, so a setup with only chat + embed never spawns rerank or + # vision. Set ``cfg.worker_pool_eager_start = false`` for headless scripts + # where mount time matters more than first-call latency. + if cfg.worker_pool_eager_start: + from contextlib import suppress + + with suppress(Exception): + _state.singleton.provider.warm_up_pool() + return _state.singleton @contextmanager @@ -292,16 +308,17 @@ def sync_vision_ocr_backend(provider: LLMProvider) -> None: from lilbee.data.ingest.types import OcrBackendName from lilbee.data.ingest.vision_ocr_backend import VisionOcrBackend - registered = OcrBackendName.LILBEE_VISION in list_ocr_backends() - if cfg.vision_model: - # Re-register so the backend always binds to the current provider. - if registered: + with _backend_bind_lock: + registered = OcrBackendName.LILBEE_VISION in list_ocr_backends() + if cfg.vision_model: + # Re-register so the backend always binds to the current provider. + if registered: + unregister_ocr_backend(OcrBackendName.LILBEE_VISION) + register_ocr_backend( + VisionOcrBackend(ocr_fn=provider.vision_ocr, model_ref_fn=lambda: cfg.vision_model) + ) + elif registered: unregister_ocr_backend(OcrBackendName.LILBEE_VISION) - register_ocr_backend( - VisionOcrBackend(ocr_fn=provider.vision_ocr, model_ref_fn=lambda: cfg.vision_model) - ) - elif registered: - unregister_ocr_backend(OcrBackendName.LILBEE_VISION) def sync_embedding_backend(provider: LLMProvider) -> None: @@ -324,11 +341,12 @@ def sync_embedding_backend(provider: LLMProvider) -> None: from lilbee.data.embedding_backend import LilbeeEmbeddingBackend from lilbee.data.ingest.types import EmbeddingBackendName - if EmbeddingBackendName.LILBEE in list_embedding_backends(): - unregister_embedding_backend(EmbeddingBackendName.LILBEE) - register_embedding_backend( - LilbeeEmbeddingBackend(embed_fn=provider.embed, dim_fn=lambda: cfg.embedding_dim) - ) + with _backend_bind_lock: + if EmbeddingBackendName.LILBEE in list_embedding_backends(): + unregister_embedding_backend(EmbeddingBackendName.LILBEE) + register_embedding_backend( + LilbeeEmbeddingBackend(embed_fn=provider.embed, dim_fn=lambda: cfg.embedding_dim) + ) def sync_tokenizer_backend(provider: LLMProvider) -> None: @@ -352,14 +370,15 @@ def sync_tokenizer_backend(provider: LLMProvider) -> None: from lilbee.data.ingest.types import TokenizerBackendName from lilbee.data.tokenizer_backend import LilbeeTokenizerBackend - registered = TokenizerBackendName.LILBEE in list_tokenizer_backends() - if cfg.token_sizing: - # Re-register so the backend always binds to the current provider. - if registered: + with _backend_bind_lock: + registered = TokenizerBackendName.LILBEE in list_tokenizer_backends() + if cfg.token_sizing: + # Re-register so the backend always binds to the current provider. + if registered: + unregister_tokenizer_backend(TokenizerBackendName.LILBEE) + register_tokenizer_backend(LilbeeTokenizerBackend(count_fn=provider.count_tokens)) + elif registered: unregister_tokenizer_backend(TokenizerBackendName.LILBEE) - register_tokenizer_backend(LilbeeTokenizerBackend(count_fn=provider.count_tokens)) - elif registered: - unregister_tokenizer_backend(TokenizerBackendName.LILBEE) def set_services(services: Services | None) -> None: diff --git a/tests/test_services.py b/tests/test_services.py index e1446341a..3749ea5b4 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -446,3 +446,49 @@ def test_no_op_when_services_uncached(self): assert services_mod.peek_services() is None reset_store() assert services_mod.peek_services() is None + + +class TestGetServicesThreadSafety: + def test_concurrent_first_touch_builds_one_container(self, monkeypatch): + """Concurrent first ``get_services()`` calls must build exactly one container. + + A duplicate concurrent build re-registers xberg's process-global backends + mid-flight, and the losing thread raises 'Embedding backend already + registered' -- surfacing as failed download tasks when several workers + first-touch services at once.""" + import threading + import time + + from lilbee.app import services as services_mod + from tests.conftest import make_mock_services + + builds: list[int] = [] + + def slow_build(config, provider=None, registry=None): + builds.append(threading.get_ident()) + time.sleep(0.05) + return make_mock_services() + + monkeypatch.setattr(services_mod, "build_services", slow_build) + monkeypatch.setattr("lilbee.app.settings.reconcile_embedding_dim", lambda registry: None) + monkeypatch.setattr(cfg, "worker_pool_eager_start", False) + services_mod._state.singleton = None + + results: list[object] = [] + errors: list[BaseException] = [] + + def touch() -> None: + try: + results.append(services_mod.get_services()) + except BaseException as exc: + errors.append(exc) + + threads = [threading.Thread(target=touch) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors + assert len(builds) == 1 + assert len({id(r) for r in results}) == 1 From 7084ea4a86a161c2c9663b3ef7d2314f8697b005 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:10:18 -0400 Subject: [PATCH 50/77] build: make the macOS floor of each binary honest and enforce it The macOS binaries claimed an 11.0 deployment target, but that only ever pinned the Nuitka launcher. Nuitka also bundles every dependency's dylib, and dyld enforces each one's minos independently, so the real floor was whatever the newest bundled dylib demanded -- and nothing checked it. The claim was already false: uv resolves wheels for the runner, so a macOS 15 image pulls numpy and scipy as macosx_14_0 and the binary could not launch below macOS 14. Each macOS asset now declares the floor it actually runs on, and a build step reads the minos of every bundled dylib and fails if any exceeds it. A dependency that raises the floor now breaks the build instead of the user's Mac. The Intel cells resolve wheels for the x86_64 target rather than the runner. numpy, scipy and pyarrow all publish older-built x86_64 wheels that run just as well on the macOS 15 image, so pinning resolution brings the floor down to 12.0 and reaches the Ivy Bridge and Haswell Macs that cap at macOS 12, which is the whole point of shipping an Intel build. arm64 stays at 14.0, where uv sync leaves it. --- .github/workflows/release.yml | 53 +++++++++++++++++++------- tools/wheel-build/check_macos_floor.sh | 45 ++++++++++++++++++++++ 2 files changed, 84 insertions(+), 14 deletions(-) create mode 100755 tools/wheel-build/check_macos_floor.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 147c3a224..af6c92f95 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -133,18 +133,25 @@ jobs: compat: true soft: true march: x86-64-v2 + # macos_floor is the OS this asset actually runs on, and every bundled + # dylib is checked against it. `uv sync` resolves wheels for the runner, + # so numpy/scipy arrive as macosx_14_0 and 14.0 is the honest floor here. - os: macos-latest asset_name: lilbee-macos-arm64 backend: metal + macos_floor: "14.0" # Intel Mac: built natively on macos-15-intel, GitHub's Intel image # (Nuitka can't cross-compile arch from the arm64 runner). Soft because # the Intel image can queue longer than the arm64 pool -- this asset must - # never block the release fan-out. Deployment target stays 11.0, so the - # AVX baseline covers every Mac that can run macOS 11+ (Ivy Bridge or newer). + # never block the release fan-out. The install below pins wheel resolution + # to the x86_64 target rather than the runner, so the deps come in at their + # oldest macOS build (pyarrow's 12.0 is the binding one) and the AVX + # baseline reaches every Ivy Bridge / Haswell Mac, which cap at macOS 12. - os: macos-15-intel asset_name: lilbee-macos-x86_64 backend: cpu soft: true + macos_floor: "12.0" # Pre-Haswell-safe Intel Mac build: same self-contained model as the # linux/windows compat cells (stock lancedb swapped for +compat, launcher # held to x86-64-v2). Covers the AVX-but-no-AVX2 tail (e.g. 2013 Mac Pro) @@ -155,6 +162,7 @@ jobs: compat: true soft: true march: x86-64-v2 + macos_floor: "12.0" - os: windows-latest asset_name: lilbee-windows-x86_64.exe backend: vulkan @@ -241,30 +249,46 @@ jobs: # Intel Mac: `uv sync` can't run here -- stock lancedb 0.33.0 ships no # macosx_x86_64 wheel. Resolve it from the +compat index (PEP 440 drop-in), - # then install the rest of [release] fresh from PyPI (xberg >=1.0.0rc22 and - # every other dep ship an x86_64 macOS wheel). CMAKE_ARGS + deployment target - # hold any throwaway llama-cpp-python sdist build at the cpu/x86_64/macOS-11 + # then install the rest of [release] fresh from PyPI. CMAKE_ARGS + deployment + # target hold any throwaway llama-cpp-python sdist build at the cpu/x86_64 # baseline; the dedicated step below builds the llama-cpp-python that actually # ships. The cell stays soft so an install failure never blocks the fan-out. + # + # --python-platform resolves wheels for the x86_64 target instead of the + # runner. Without it uv picks the newest wheel the macOS 15 image accepts + # (numpy/scipy macosx_14_0) and bundles a binary that dies on macOS 12-13, + # even though both ship older-built wheels that run just as well here. - name: Install dependencies (Intel Mac) if: matrix.os == 'macos-15-intel' shell: bash env: - MACOSX_DEPLOYMENT_TARGET: "11.0" + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_floor }} run: | set -euxo pipefail eval "$(BACKEND=cpu TARGET_ARCH=x86_64 tools/wheel-build/cmake_args.sh)" export CMAKE_ARGS uv venv - uv pip install --extra-index-url https://lilbee.sh/compat/ \ + uv pip install --python-platform x86_64-apple-darwin \ + --extra-index-url https://lilbee.sh/compat/ \ lancedb==0.33.0+compat - uv pip install '.[release]' 'nuitka[onefile]>=4.0,<5' pip + uv pip install --python-platform x86_64-apple-darwin \ + '.[release]' 'nuitka[onefile]>=4.0,<5' pip - name: Verify extras installed in build venv shell: bash # --no-sync is load-bearing: re-syncing reinstalls python-multipart and shadows the shim. run: uv run --no-sync python -c "import crawl4ai, litellm, spacy, graspologic_native; print('extras OK')" + # Nuitka bundles these dylibs, and dyld enforces each one's minos separately + # from the launcher's, so one dep built on a newer runner silently ships a + # binary that dies at load. Fail here rather than on the user's Mac. + - name: Verify macOS floor + if: runner.os == 'macOS' + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_floor }} + run: bash tools/wheel-build/check_macos_floor.sh + # compat cell only: swap the synced stock lancedb for the pre-Haswell build. # --no-deps leaves the rest of the venv intact, so nothing leaves this cell. # Pinned to the latest stable lancedb for Python (0.33), served by the fork as @@ -288,18 +312,19 @@ jobs: # engine cache separate from bare-runner builds. cache-env: ${{ matrix.container || matrix.os }} env: - MACOSX_DEPLOYMENT_TARGET: "11.0" + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_floor }} - name: Build executable shell: bash env: ASSET_NAME: ${{ matrix.asset_name }} PRODUCT_VERSION: ${{ env.LILBEE_NUITKA_VERSION }} - # Pin Nuitka's clang output to macOS 11.0. Unset, clang defaults the - # deployment target to the build host (macOS 15 on the Intel image), which - # would stamp the launcher minos=15 and refuse to launch on macOS 11-14 even - # though every bundled dylib targets 11.0. No-op off macOS. - MACOSX_DEPLOYMENT_TARGET: "11.0" + # Pin Nuitka's clang output to this asset's floor. Unset, clang defaults the + # deployment target to the build host (macOS 15), which would stamp the + # launcher minos=15 and refuse to launch below it. This only pins the + # launcher; the bundled dylibs carry their own minos, which is what the + # "Verify macOS floor" step above checks. No-op off macOS. + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_floor }} # compat cell only: hold Nuitka's C output to the pre-Haswell baseline so # the launcher layer (not just lancedb) runs on old silicon. Empty elsewhere. CFLAGS: ${{ matrix.march && format('-march={0}', matrix.march) || '' }} diff --git a/tools/wheel-build/check_macos_floor.sh b/tools/wheel-build/check_macos_floor.sh new file mode 100755 index 000000000..a5191b38f --- /dev/null +++ b/tools/wheel-build/check_macos_floor.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Fail if any native library in the build venv is built for a newer macOS than +# the deployment target the standalone binary claims. +# +# Nuitka bundles these dylibs into the onefile, and dyld enforces each one's +# LC_BUILD_VERSION minos independently of the launcher's. A dependency built on +# a macOS 15 runner therefore ships a binary that dies at load on macOS 11-14 +# even though the launcher itself is pinned to 11.0 -- the failure lands on the +# user, not on the build, so nothing here catches it without an explicit check. +# +# Usage: +# MACOSX_DEPLOYMENT_TARGET=11.0 tools/wheel-build/check_macos_floor.sh +# No-op off macOS. +set -euo pipefail + +[[ "$(uname -s)" == "Darwin" ]] || exit 0 + +FLOOR="${MACOSX_DEPLOYMENT_TARGET:?MACOSX_DEPLOYMENT_TARGET is required}" +SITE_PACKAGES="$(uv run --no-sync python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" + +# Sort -V orders versions, so the greater of {floor, minos} is the last line; a +# minos that is not the floor and sorts above it is newer than the floor. +exceeds_floor() { + [[ "$1" != "$FLOOR" && "$(printf '%s\n%s\n' "$FLOOR" "$1" | sort -V | tail -1)" == "$1" ]] +} + +offenders="" +while IFS= read -r lib; do + # A Mach-O may carry no LC_BUILD_VERSION (older linkers emit LC_VERSION_MIN_MACOSX + # instead, and text files caught by the glob carry neither); read both, skip neither-case. + minos="$(otool -l "$lib" 2>/dev/null | awk '/LC_BUILD_VERSION|LC_VERSION_MIN_MACOSX/{f=1} f&&/minos|version/{print $2; exit}')" + [[ -n "$minos" ]] || continue + if exceeds_floor "$minos"; then + offenders+=" $minos ${lib#"$SITE_PACKAGES"/}"$'\n' + fi +done < <(find "$SITE_PACKAGES" \( -name '*.so' -o -name '*.dylib' \) -type f) + +if [[ -n "$offenders" ]]; then + echo "ERROR: these bundled libraries require a newer macOS than the ${FLOOR} floor" >&2 + echo " this binary claims, so it would fail to launch below their minos:" >&2 + printf '%s' "$offenders" >&2 + exit 1 +fi + +echo "macOS floor OK: every bundled library loads on ${FLOOR}+" From d77ae8803d646bed5b00dfbbbdbf3b53480477b5 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 10 Jul 2026 11:08:05 -0400 Subject: [PATCH 51/77] retrieval: canonical score fusion and intent routing --- docs/architecture.md | 97 +++-- docs/usage.md | 20 +- src/lilbee/app/settings_map.py | 24 ++ src/lilbee/core/config/model.py | 39 +- src/lilbee/data/store/core.py | 184 +++++++-- src/lilbee/data/store/fusion.py | 77 ++++ src/lilbee/data/store/ranking.py | 12 +- src/lilbee/data/store/types.py | 18 +- src/lilbee/retrieval/concepts/graph.py | 8 +- src/lilbee/retrieval/query/dedup.py | 81 ++-- src/lilbee/retrieval/query/expansion.py | 15 +- src/lilbee/retrieval/query/formatting.py | 89 +++- src/lilbee/retrieval/query/intent.py | 157 +++++++ src/lilbee/retrieval/query/searcher.py | 242 +++++++++-- src/lilbee/server/handlers/rag.py | 108 +++-- src/lilbee/server/models.py | 3 + tests/test_concepts.py | 17 + tests/test_intent.py | 139 +++++++ tests/test_query.py | 497 +++++++++++++++++++++-- tests/test_server_handlers.py | 30 ++ tests/test_server_litestar.py | 2 + tests/test_store.py | 79 +++- tests/test_store_fusion.py | 121 ++++++ 23 files changed, 1811 insertions(+), 248 deletions(-) create mode 100644 src/lilbee/data/store/fusion.py create mode 100644 src/lilbee/retrieval/query/intent.py create mode 100644 tests/test_intent.py create mode 100644 tests/test_store_fusion.py diff --git a/docs/architecture.md b/docs/architecture.md index e2e6c9be4..208821217 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -534,55 +534,77 @@ rather than a 500 from the server. ## Search Pipeline -This is the core of lilbee's retrieval quality. The pipeline applies techniques progressively: expensive operations are skipped when simpler ones produce confident results. +This is the core of lilbee's retrieval quality. Questions are routed by shape before retrieval runs, every result carries one canonical relevance score, and expensive operations are skipped when simpler ones produce confident results. ```mermaid flowchart TD - Q[User Query] --> SM{Structured Mode?} + Q[User Query] --> HR{Chat history?} + HR -->|Yes| COND[Condense follow-up into standalone query] + HR -->|No| ROUTE + COND --> ROUTE{Intent} + ROUTE -->|count-shaped| SCAN[Exact full-corpus scan → direct answer] + ROUTE -->|names one document| DOC[Document's own chunks, in order] + ROUTE -->|topical| SM{Structured Mode?} SM -->|term: prefix| BM25[BM25 Keyword Search] SM -->|vec: prefix| VEC[Vector Search] SM -->|hyde: prefix| HYDE_M[HyDE → Embed → Search] - SM -->|No prefix| STD[Standard Pipeline] + SM -->|No prefix| PROBE[BM25 Confidence Probe] - STD --> TF{Temporal Keywords?} - TF -->|Yes| TPARSE[Parse Date Range] - TF -->|No| PROBE - - TPARSE --> PROBE[BM25 Confidence Probe] - PROBE --> CONF{Score ≥ 0.8 AND gap ≥ 0.15?} - CONF -->|Yes| HYBRID[Hybrid Search Only] + PROBE --> CONF{Confident AND separated?} + CONF -->|Yes| DUAL[Dual-Arm Retrieval] CONF -->|No| EXPAND[LLM Query Expansion] EXPAND --> GEXP[+ Graph Expansion] GEXP --> GUARD[Guardrails: embedding cosine similarity] GUARD --> MULTI[Multi-Query Search + Merge] - MULTI --> HYBRID - - HYBRID --> CBOOST[Concept Boost] - CBOOST --> ADAPT[Adaptive Distance Filter] - ADAPT --> MMR[MMR Diversity] - MMR --> RERANK{Reranker Model?} + MULTI --> DUAL + + DUAL --> FUSE[Score Fusion → canonical score] + FUSE --> MMR[MMR Diversity on canonical score] + MMR --> CBOOST[Concept Boost] + CBOOST --> GSORT[Global re-sort by score] + GSORT --> ABST{All below min_relevance_score?} + ABST -->|Yes| REFUSE[Grounded refusal] + ABST -->|No| RERANK{Reranker Model?} RERANK -->|Yes| XENC[Cross-Encoder Rerank] RERANK -->|No| DIV XENC --> DIV[Source Diversity Cap] DIV --> TFILTER{Temporal Filter?} - TFILTER -->|Yes| TFILT[Filter by Date + Recency Sort] + TFILTER -->|Yes| TFILT[Filter by Date] TFILTER -->|No| CTX TFILT --> CTX[Adaptive Context Selection] - CTX --> BUILD[Build Context → LLM Generation] + CTX --> BUILD[Context with provenance headers → LLM Generation] ``` ### Technique Reference -#### Hybrid Search (BM25 + Vector + RRF) -**Always on.** Combines keyword matching (BM25 via LanceDB FTS) with semantic similarity (vector cosine distance), fused via Reciprocal Rank Fusion. +#### Intent Routing +**On by default** (`LILBEE_INTENT_ROUTING`). Three question shapes reach retrieval, and only one of them is a similarity problem, so questions are routed before top-k search runs. Detection is deterministic and conservative: anything unrecognized takes the topical path unchanged. + +- **Known-item**: a question naming a document (a filename, a quoted title, "document 482") resolves the reference against source metadata; a reference matching exactly one source returns that document's chunks in document order at full confidence. Similarity search retrieves neighbors of the question's *wording*, which for "summarize survey_482.pdf" is mostly noise. +- **Aggregate**: "how many documents mention X" runs an exact scan over every chunk (streamed Arrow batches, flat memory) and answers with real numbers, no LLM involved. A count is a corpus property; the top 20 of half a million chunks structurally cannot count anything, and the model's only honest move was to hedge. Counts that need typed records the store does not hold are declined with a precise statement of what is countable. +- Every answering surface consults the same router (`Searcher.route_direct_answer`): CLI and TUI via ask, and each HTTP handler directly. + +#### History Condensation +**On by default** (`LILBEE_HISTORY_REWRITE`). A follow-up question is condensed into a standalone retrieval query using the recent chat history (one small LLM call, skipped when there is no history). Retrieval sees only the query text: without this, "what about his brother?" is embedded and BM25-matched with its pronouns. The user's original wording still reaches the answering prompt. + +#### Hybrid Search (Dual-Arm Score Fusion) +**Always on.** Retrieves the BM25 arm (LanceDB FTS) and the vector arm independently, each overfetched well past `top_k` (`candidate_multiplier × top_k`, floored at `fusion_overfetch_floor`, default 50 rows per arm), then fuses by convex combination into one canonical [0, 1] score: -- **Paper**: Cormack, Clarke & Büttcher 2009, "[Reciprocal Rank Fusion outperforms Condorcet and individual Rank Learning Methods](https://dl.acm.org/doi/10.1145/1571941.1572114)" -- **Tradeoff**: ~5ms overhead vs vector-only search. Worth it because BM25 catches exact keyword matches that vectors miss (e.g. searching for "NavigationServer2D" needs exact string matching, not semantic similarity). -- **When it helps**: queries with specific terms, function names, error messages, exact phrases. +``` +score = fusion_alpha × vector_similarity + (1 - fusion_alpha) × normalized_bm25 +``` + +Vector similarity is clamped `1 - cosine_distance` (an absolute signal); BM25 is normalized against the top score of its list. `fusion_alpha` defaults to 0.6. + +- **Why score fusion and not rank fusion**: Reciprocal Rank Fusion (the previous mechanism) discards score magnitude by construction, so it cannot tell an arm that was certain from an arm that was guessing. One strong lexical hit for an identifier query drowned under mediocre dense neighbors, fetched from a fusion pool that was only `top_k` deep per arm. Score-aware fusion keeps the certainty, and the overfetch makes rows ranked just past `top_k` in both arms visible to fusion at all. (On the rank-vs-score question, see Bruch et al. 2024, "[An Analysis of Fusion Functions for Hybrid Retrieval](https://dl.acm.org/doi/10.1145/3596512)".) +- **Canonical score**: every search path sets `SearchChunk.score`, and every downstream stage (sorting, filtering, MMR, greedy set cover, concept boost, reranker blending) compares only that field. `distance`, `bm25_score`, and the legacy RRF `relevance_score` remain as provenance. +- **Abstention**: because the canonical score is [0, 1] with real meaning, `min_relevance_score` is a usable floor: when every retrieved chunk falls below it, ask refuses instead of feeding noise as context. RRF magnitudes (~0.016-0.033 total range) made any threshold meaningless. +- **max_distance** applies to rows whose *only* signal is a far vector match; a row the BM25 arm also matched keeps its standing regardless of distance, since dropping it would re-bury exactly the identifier hits fusion exists to preserve. +- **When it helps**: queries with specific terms, function names, error messages, exact phrases, document identifiers. #### MMR Diversity -**Always on.** Maximal Marginal Relevance prevents near-duplicate chunks from filling all result slots. +**Always on.** Maximal Marginal Relevance prevents near-duplicate chunks from filling all result slots. Runs on the fused candidate pool, with the canonical score as the relevance term, so diversity selection cannot re-demote a lexically-certain hit whose vector similarity happens to be weak. - **Paper**: Carbonell & Goldstein 1998, "[The Use of MMR, Diversity-Based Reranking](https://dl.acm.org/doi/10.1145/290941.291025)" - **Default**: λ=0.5 (equal weight to relevance and diversity). This is the standard default from the original paper. @@ -593,7 +615,7 @@ flowchart TD **Always on.** Caps results per source document so one large file doesn't dominate all top-k slots. - **Paper**: Zhai 2008, "[Towards a Game-Theoretic Framework for Information Retrieval](https://dl.acm.org/doi/10.1007/978-3-540-78646-7_13)" -- **Default**: 3 chunks per source. Ensures at least 2 different documents appear in top-5 results. +- **Default**: 5 chunks per source (`LILBEE_DIVERSITY_MAX_PER_SOURCE`). - **Tradeoff**: lower cap = more diverse sources but may miss relevant sections from a single comprehensive document. #### Query Expansion @@ -605,14 +627,14 @@ flowchart TD - **When it helps**: queries using different terminology than the indexed documents. E.g. user asks "how to deploy" but the docs say "installation steps". #### Confidence-Based Expansion Skip -**On by default.** Before running the expensive LLM expansion call, does a quick BM25 probe. If the top BM25 result is highly confident, expansion is skipped entirely. +**On by default.** Before running the expensive LLM expansion call, does a quick BM25 probe. If the top BM25 result is highly confident and clearly separated from the runner-up, expansion is skipped entirely. - **Technique**: early termination based on BM25 score distribution -- **Default threshold**: 0.80 (90th percentile of sigmoid-normalized BM25 scores) -- **Default gap**: 0.15 (top-1 must be clearly separated from top-2) -- **Threshold derivation**: BM25 scores are normalized via sigmoid centered at ~0.5. Scores above 0.8 represent strong keyword matches. The gap ensures the match isn't ambiguous. +- **Default threshold**: 0.80 in saturating confidence space `s / (s + 5)`, which corresponds to a raw BM25 score of 20 +- **Default gap**: 0.15 as the *relative* raw-score gap, `(top - second) / top` +- **Why not a sigmoid**: a plain sigmoid saturates at ~0.99 for any raw score above 5, so the gap between two strong scores compressed to under 0.01 and the skip condition was unsatisfiable exactly when the lexical arm was most certain. The saturating ratio keeps strong scores distinguishable and the relative gap compares where the information actually is, in raw score space. - **Tradeoff**: higher threshold = expansion runs more often (better recall, more latency). Lower = expansion skipped more (faster, may miss some results). -- **Caveat**: these are starting defaults. Calibrate for your library using RAGAS evaluation metrics. +- **Caveat**: these are starting defaults; calibrate for your corpus. #### Expansion Guardrails **On by default.** Validates LLM-generated query variants to prevent drift. @@ -623,11 +645,11 @@ flowchart TD - **Tradeoff**: guardrails may filter out creative but valid variants. Disable via `LILBEE_EXPANSION_GUARDRAILS=false` if recall is more important than precision. #### HyDE (Hypothetical Document Embeddings) -**Off by default.** Generates a hypothetical excerpt (50-100 words) that reads like a real document answering the query, embeds it, and searches with it alongside the original query vector. +**Off by default.** Generates a hypothetical excerpt (50-100 words) that reads like a real document answering the query, embeds it, and searches with it alongside the original query vector. Reasoning-model output is stripped before embedding, so a think-block never stands in for the passage. - **Paper**: Gao et al. 2022, "[Precise Zero-Shot Dense Retrieval without Relevance Labels](https://arxiv.org/abs/2212.10496)" - **Cost**: 1 additional LLM call + 1 embedding (~500ms total) -- **Default weight**: 0.7x (hypothetical results are discounted because they're fabricated: they approximate the answer space but aren't based on real content) +- **Default weight**: 0.7, applied multiplicatively to the canonical score (hypothetical results are discounted because they're fabricated: they approximate the answer space but aren't based on real content; 1.0 trusts them as much as direct hits) - **When it helps**: vague or short queries where the user's terminology doesn't match the indexed documents. E.g. "how does the thing work" where the "thing" is described with specific technical vocabulary in the docs. - **When to skip**: factual lookups, keyword-heavy queries, or when latency matters. @@ -635,7 +657,7 @@ flowchart TD **On by default.** At index time, extracts noun phrases from each chunk via spaCy, builds a co-occurrence graph weighted by Positive Pointwise Mutual Information (PPMI), and clusters concepts with the Leiden algorithm. Zero LLM calls at index or query time. Two query-time effects: -- **Concept boost**: for each search result, counts concept overlap between the query's noun phrases and the chunk's concepts. Score adjusted by `overlap_ratio × concept_boost_weight` (default 0.3). Only promotes, never demotes. +- **Concept boost**: for each search result, counts concept overlap between the query's noun phrases and the chunk's concepts. The canonical score is raised by `overlap_ratio × concept_boost_weight` (default 0.3, clamped at 1.0), so the boost weight is directly comparable to a fusion arm's weight. Only promotes, never demotes. - **Graph expansion**: traverses the co-occurrence graph (1 hop BFS) to find concepts related to the query. These supplement LLM-generated expansion variants and go through the same drift guardrails. - **Inspiration**: Microsoft Research 2024-2025, "[LazyGraphRAG](https://www.microsoft.com/en-us/research/blog/lazygraphrag-setting-a-new-standard-for-quality-and-cost/)". NLP concept extraction at index time, defer reasoning to query time. @@ -667,13 +689,16 @@ Two query-time effects: - **When it helps**: novel queries or small indexes where strict distance thresholds would return empty results. #### Adaptive Context Selection -**On by default.** After search results are ranked, selects which chunks to include as LLM context based on query term coverage rather than just taking the top-k. When cross-encoder reranking is active, the reranked order is the more precise signal and replaces this pass. +**On by default.** After search results are ranked, selects which chunks to include as LLM context based on query term coverage rather than just taking the top-k, with each chunk's IDF gain weighted by its canonical score. When cross-encoder reranking is active, the reranked order is the more precise signal and replaces this pass. - **Technique**: greedy set-cover approximation -- **Algorithm**: tokenize query into terms, greedily select chunks that add the most uncovered terms, stop when coverage reaches 100% or marginal gain drops below 5% -- **Default max sources**: 5 chunks +- **Algorithm**: tokenize query into terms, greedily select chunks that add the most uncovered term weight, scaled by canonical relevance +- **Default max sources**: 8 chunks (`LILBEE_MAX_CONTEXT_SOURCES`) - **When it helps**: multi-faceted queries like "compare X and Y" where top-k might only cover X but context selection ensures Y is also represented. +#### Context Provenance Headers +**Always on.** Each context block opens with a one-line header naming its source and page or line span (`[3] (survey_report.pdf, pages 3-4)`), so the answering model can attribute claims to a named document, notice two chunks share a source, and confirm it is reading the document the user asked about. Citation markers in answers are parsed as `[1]`, `[1, 2]`, and bounded ranges like `[1-3]`. + #### Temporal Filtering **On by default, activates only when temporal keywords are detected in the query.** diff --git a/docs/usage.md b/docs/usage.md index f22e9e581..5c9b7d164 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -778,11 +778,16 @@ reason the defaults are the defaults. | Variable | Default | Description | |----------|---------|-------------| -| `LILBEE_EXPANSION_SKIP_THRESHOLD` | `0.8` | BM25 confidence threshold above which query expansion is skipped (90th-percentile sigmoid-normalized score) | -| `LILBEE_EXPANSION_SKIP_GAP` | `0.15` | Minimum score gap between top-1 and top-2 for expansion to skip (ensures the match is unambiguous) | +| `LILBEE_EXPANSION_SKIP_THRESHOLD` | `0.8` | BM25 confidence above which query expansion is skipped; confidence is `s / (s + 5)`, so 0.8 = raw score 20 | +| `LILBEE_EXPANSION_SKIP_GAP` | `0.15` | Minimum relative raw-score gap between top-1 and top-2, `(top - second) / top`, for expansion to skip | | `LILBEE_EXPANSION_GUARDRAILS` | `true` | Filter expansion variants whose embedding drifts too far from the original query | | `LILBEE_EXPANSION_SIMILARITY_THRESHOLD` | `0.5` | Minimum query-variant cosine similarity to survive the guardrail | -| `LILBEE_CANDIDATE_MULTIPLIER` | `3` | Extra candidates to retrieve before MMR reranking | +| `LILBEE_CANDIDATE_MULTIPLIER` | `3` | Per-arm retrieval depth as a multiple of top_k (hybrid overfetch; also the vector-only MMR pool) | +| `LILBEE_FUSION_OVERFETCH_FLOOR` | `50` | Minimum rows fetched per retrieval arm before fusion, regardless of top_k | +| `LILBEE_FUSION_ALPHA` | `0.6` | Vector-arm weight in hybrid score fusion; the BM25 arm gets the complement | +| `LILBEE_MIN_RELEVANCE_SCORE` | `0.0` | Abstention floor against the canonical [0, 1] relevance score; when every result falls below it, ask refuses instead of answering from noise | +| `LILBEE_HISTORY_REWRITE` | `true` | Condense follow-up questions into standalone retrieval queries using chat history | +| `LILBEE_INTENT_ROUTING` | `true` | Route document-name lookups to exact retrieval and count questions to a full-corpus scan | ## Optional extras @@ -1035,10 +1040,11 @@ the top results. Unlike the extras above, no extra install is required; reranking is off by default and turns on as soon as you set `LILBEE_RERANKER_MODEL` (or pick a reranker from `/settings`). -**What it does:** After the hybrid search pipeline (BM25 + vector + RRF) -returns candidates, a GGUF cross-encoder scores each `(query, chunk)` pair and -results are blended with position-aware weights. Top-ranked candidates keep -more of the original ranking; lower-ranked candidates trust the reranker more. +**What it does:** After hybrid search (BM25 and vector arms fused into one +canonical relevance score) returns candidates, a GGUF cross-encoder scores +each `(query, chunk)` pair and results are blended with position-aware +weights. Top-ranked candidates keep more of the original ranking; +lower-ranked candidates trust the reranker more. **When to use it:** When you need high-precision answers and are willing to trade roughly 200 to 500 ms per query. Most useful with large candidate sets diff --git a/src/lilbee/app/settings_map.py b/src/lilbee/app/settings_map.py index 7049884fb..2827e8fff 100644 --- a/src/lilbee/app/settings_map.py +++ b/src/lilbee/app/settings_map.py @@ -793,6 +793,30 @@ def get_default(key: str) -> object: group=SettingGroup.RETRIEVAL, help_text="Candidate-pool multiplier over top_k before reranking", ), + "fusion_alpha": SettingDef( + float, + nullable=False, + group=SettingGroup.RETRIEVAL, + help_text="Vector-arm weight in hybrid fusion (1.0 = pure vector, 0.0 = pure lexical)", + ), + "fusion_overfetch_floor": SettingDef( + int, + nullable=False, + group=SettingGroup.RETRIEVAL, + help_text="Minimum rows fetched per retrieval arm before fusion", + ), + "history_rewrite": SettingDef( + bool, + nullable=False, + group=SettingGroup.RETRIEVAL, + help_text="Rewrite follow-ups into standalone retrieval queries using chat history", + ), + "intent_routing": SettingDef( + bool, + nullable=False, + group=SettingGroup.RETRIEVAL, + help_text="Route document-name lookups to exact retrieval, count questions to a scan", + ), "ann_index_threshold": SettingDef( int, nullable=False, diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index 32c280c42..1e108ebce 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -95,11 +95,10 @@ class Config(BaseSettings): max_embed_chars: int = Field(default=2000, ge=1) top_k: int = ConfigField(default=12, ge=1, writable=True) max_distance: float = ConfigField(default=0.75, ge=0.0, writable=True) - # Floor for hybrid-search relevance scores (0.0 = no filtering). lilbee - # surfaces LanceDB's raw RRF sum, not a normalized score: with K=60 a - # chunk ranked first in both the vector and FTS lists tops out near - # 1/61 + 1/61 ~= 0.033, so any positive floor above that silently drops - # every result. Keep this at 0.0 unless the RRF scores are normalized first. + # Abstention floor against the canonical [0, 1] relevance score + # (0.0 = no filtering). When every retrieved chunk falls below it, ask + # refuses instead of feeding noise as context; useful values start + # around 0.05-0.15 depending on fusion_alpha. min_relevance_score: float = ConfigField(default=0.0, ge=0.0, writable=True) adaptive_threshold: bool = Field(default=False) rag_system_prompt: str = ConfigField( @@ -184,14 +183,37 @@ class Config(BaseSettings): # (Carbonell & Goldstein 1998). mmr_lambda: float = ConfigField(default=0.5, ge=0.0, le=1.0, writable=True) - # Extra candidates retrieved for MMR reranking (multiplies top_k). + # Per-arm retrieval depth as a multiple of top_k: hybrid overfetches each + # arm this deep (floored at 50) before fusion, and the vector-only path + # retrieves this many extra candidates for MMR reranking. candidate_multiplier: int = ConfigField(default=3, ge=1, writable=True) + # Vector-arm weight in hybrid score fusion; the BM25 arm gets the + # complement. 1.0 = pure vector, 0.0 = pure lexical. + fusion_alpha: float = ConfigField(default=0.6, ge=0.0, le=1.0, writable=True) + + # Minimum rows fetched per arm before fusion, regardless of top_k and + # candidate_multiplier. Answer-level sweeps found deeper pools dilute the + # fused top-k, so this is tunable rather than fixed. + fusion_overfetch_floor: int = ConfigField(default=50, ge=1, writable=True) + # Chunk count at/above which sync builds an approximate (ANN) vector index # so search stays fast at millions of vectors. Below this, search uses exact # flat scan (faster and exact for small vaults). 0 disables the ANN index. ann_index_threshold: int = ConfigField(default=50_000, ge=0, writable=True) + # Condense a follow-up question into a standalone retrieval query using + # the chat history (one LLM call; skipped when there is no history). + # Without it, "what about his brother?" is embedded and BM25-matched + # with its pronouns. + history_rewrite: bool = ConfigField(default=True, writable=True) + + # Route questions by shape before top-k retrieval: a question naming a + # document resolves to that document's chunks; a count-shaped question + # runs a full-corpus scan (a count is a corpus property top-k cannot + # answer). Unrecognized shapes take the topical path unchanged. + intent_routing: bool = ConfigField(default=True, writable=True) + # LLM-generated alternative queries for expansion. 0 disables. query_expansion_count: int = ConfigField(default=3, ge=0, writable=True) @@ -209,10 +231,11 @@ class Config(BaseSettings): # Min cosine similarity between question and variant embeddings. expansion_similarity_threshold: float = ConfigField(default=0.5, ge=0.0, le=1.0, writable=True) - # Sigmoid-normalized BM25 score above which query expansion is skipped. + # Saturating BM25 confidence (s / (s + 5)) above which query expansion is + # skipped; 0.8 corresponds to a raw BM25 score of 20. expansion_skip_threshold: float = Field(default=0.8, ge=0.0, le=1.0) - # Min BM25 top-1 vs top-2 gap to skip expansion. + # Min relative BM25 top-1 vs top-2 gap ((top - second) / top) to skip expansion. expansion_skip_gap: float = Field(default=0.15, ge=0.0, le=1.0) # Chunks included in LLM context after adaptive selection. diff --git a/src/lilbee/data/store/core.py b/src/lilbee/data/store/core.py index e7f4a0523..43d18b624 100644 --- a/src/lilbee/data/store/core.py +++ b/src/lilbee/data/store/core.py @@ -26,6 +26,7 @@ from lilbee.core.security import validate_path_within from lilbee.runtime.lock import LOCK_TIMEOUT, write_lock +from .fusion import fuse_arms, normalized_bm25, vector_similarity from .lance_helpers import ( _chunk_type_predicate, _has_fts_index, @@ -73,32 +74,22 @@ BATCH_LOCK_TIMEOUT = 120.0 -def _hybrid_search( - table: lancedb.table.Table, - query_text: str, - query_vector: list[float], - top_k: int, - chunk_type: ChunkType | None = None, +def _drop_unsupported_far_rows( + results: list[SearchChunk], max_distance: float ) -> list[SearchChunk]: - """Run hybrid (vector + FTS) search with RRF reranking. + """Apply ``max_distance`` to rows whose only signal is the vector arm. - When ``chunk_type`` is set, the predicate is pushed into the query so - the limit applies *after* the type filter. Post-filtering would - silently starve wiki-only queries whose matches live past the top-K - hybrid window. + A row the BM25 arm also matched keeps lexical support regardless of its + vector distance; dropping it on distance alone would re-bury exactly the + identifier hits score fusion exists to preserve. """ - from lancedb.rerankers import RRFReranker - - query = ( - table.search(query_type="hybrid") - .vector(query_vector) - .text(query_text) - .rerank(RRFReranker()) - ) - if chunk_type: - query = query.where(_chunk_type_predicate(chunk_type)) - rows = query.limit(top_k).to_list() - return [SearchChunk(**r) for r in rows] + if max_distance <= 0: + return results + return [ + r + for r in results + if r.bm25_score is not None or r.distance is None or r.distance <= max_distance + ] _MAX_THRESHOLD = 1.0 @@ -131,6 +122,10 @@ def _hybrid_search( # replace would join millions of filenames into one delete predicate. _SOURCE_STAT_BATCH_ROWS = 2000 +# Rows per Arrow batch when the aggregate scan walks the whole chunks table; +# bounds the decoded-text working set while the scan stays columnar. +_TERM_SCAN_BATCH_ROWS = 20_000 + def _ann_nprobes(row_count: int) -> int: """Partitions to probe: a fixed fraction of the IVF partition count (~sqrt(N)), floored.""" @@ -495,7 +490,11 @@ def bm25_probe( if chunk_type: query = query.where(_chunk_type_predicate(chunk_type)) rows = query.limit(top_k).to_list() - return [SearchChunk(**r) for r in rows] + results = [SearchChunk(**r) for r in rows] + norms = normalized_bm25([r.bm25_score or 0.0 for r in results]) + return [ + r.model_copy(update={"score": norm}) for r, norm in zip(results, norms, strict=True) + ] except Exception: log.debug("BM25 probe failed", exc_info=True) return [] @@ -533,20 +532,17 @@ def search( if query_text and self._fts_ready: try: - return _hybrid_search(table, query_text, query_vector, top_k, chunk_type) + return self._hybrid_search( + table, query_text, query_vector, top_k, max_distance, chunk_type + ) except Exception: - log.debug("Hybrid search failed, falling back to vector-only", exc_info=True) + # Falling back changes recall characteristics for the query; + # a corpus-wide FTS breakage must not present as silence. + log.warning("Hybrid search failed, falling back to vector-only", exc_info=True) - candidate_k = top_k * self._config.candidate_multiplier - query = table.search(query_vector).metric(_VECTOR_METRIC).limit(candidate_k) - if _has_vector_index(table): - # IVF_PQ is lossy; probe more partitions and refine against full - # vectors so recall stays close to the exact flat scan. - nprobes = _ann_nprobes(table.count_rows()) - query = query.nprobes(nprobes).refine_factor(_ANN_REFINE_FACTOR) - if chunk_type: - query = query.where(_chunk_type_predicate(chunk_type)) - rows = query.to_list() + rows = self._vector_arm( + table, query_vector, top_k * self._config.candidate_multiplier, chunk_type + ) log.debug( "Vector search: query=%r, candidates=%d, max_distance=%.2f", query_text or "vector-only", @@ -554,12 +550,86 @@ def search( max_distance, ) if rows: - # LanceDB names the similarity column "_distance"; "distance" would - # always miss and log 0. - distances = [r.get("_distance", 0) for r in rows[:5]] - log.debug("Top 5 distances: %s", distances) - results = [SearchChunk(**r) for r in rows] - return self._filter_and_rerank(results, query_vector, top_k, max_distance) + log.debug("Top 5 distances: %s", [r.distance for r in rows[:5]]) + results = self._filter_and_rerank(rows, query_vector, top_k, max_distance) + return [ + r.model_copy( + update={"score": vector_similarity(r.distance) if r.distance is not None else 0.0} + ) + for r in results + ] + + def _vector_arm( + self, + table: lancedb.table.Table, + query_vector: list[float], + limit: int, + chunk_type: ChunkType | None, + ) -> list[SearchChunk]: + """Vector-arm candidates with the ANN recall recovery applied. + + When ``chunk_type`` is set, the predicate is pushed into the query so + the limit applies *after* the type filter; post-filtering would + silently starve wiki-only queries whose matches live past the window. + """ + query = table.search(query_vector).metric(_VECTOR_METRIC).limit(limit) + if _has_vector_index(table): + # IVF_PQ is lossy; probe more partitions and refine against full + # vectors so recall stays close to the exact flat scan. + query = query.nprobes(_ann_nprobes(table.count_rows())) + query = query.refine_factor(_ANN_REFINE_FACTOR) + if chunk_type: + query = query.where(_chunk_type_predicate(chunk_type)) + return [SearchChunk(**r) for r in query.to_list()] + + def _fts_arm( + self, + table: lancedb.table.Table, + query_text: str, + limit: int, + chunk_type: ChunkType | None, + ) -> list[SearchChunk]: + """BM25-arm candidates.""" + query = table.search(query_text, query_type="fts").limit(limit) + if chunk_type: + query = query.where(_chunk_type_predicate(chunk_type)) + return [SearchChunk(**r) for r in query.to_list()] + + def _hybrid_search( + self, + table: lancedb.table.Table, + query_text: str, + query_vector: list[float], + top_k: int, + max_distance: float, + chunk_type: ChunkType | None = None, + ) -> list[SearchChunk]: + """Dual-arm retrieval fused by score, then MMR-selected down to top_k. + + Each arm is overfetched well past ``top_k`` so fusion sees rows ranked + just outside the final window in both arms; the previous rank-fused + path capped each arm at ``top_k``, hiding those rows entirely. MMR + runs on the canonical fused score, so lexical-only hits keep the + standing fusion gave them. + """ + overfetch = max( + top_k * self._config.candidate_multiplier, self._config.fusion_overfetch_floor + ) + fused = fuse_arms( + self._vector_arm(table, query_vector, overfetch, chunk_type), + self._fts_arm(table, query_text, overfetch, chunk_type), + self._config.fusion_alpha, + ) + fused = _drop_unsupported_far_rows(fused, max_distance) + if len(fused) > top_k: + fused = mmr_rerank( + query_vector, + fused, + top_k, + self._config.mmr_lambda, + relevance_scores=[r.score or 0.0 for r in fused], + ) + return fused[:top_k] def _filter_and_rerank( self, @@ -620,6 +690,36 @@ def _fixed_filter(self, results: list[SearchChunk], threshold: float) -> list[Se """Simple fixed threshold filter - keep only results within distance threshold.""" return [r for r in results if _get_distance(r) <= threshold] + def count_term_mentions(self, term: str) -> tuple[int, int]: + """(matching chunks, distinct matching sources) for a case-insensitive + substring scan of the WHOLE chunks table. + + This is deliberately a full scan, not a top-k search: a count is a + corpus property, and any retrieval shortcut undercounts it. Streaming + Arrow batches keeps the working set to one batch of text at a time, + so cost is linear in corpus size and memory stays flat. + """ + table = self.open_table(CHUNKS_TABLE) + if table is None: + return 0, 0 + needle = term.lower() + chunk_hits = 0 + sources: set[str] = set() + arrow = table.to_arrow().select(["source", "chunk"]) + for batch in arrow.to_batches(max_chunksize=_TERM_SCAN_BATCH_ROWS): + texts = batch.column("chunk").to_pylist() + names = batch.column("source").to_pylist() + for name, text in zip(names, texts, strict=True): + if text and needle in text.lower(): + chunk_hits += 1 + sources.add(name) + return chunk_hits, len(sources) + + def count_chunks(self) -> int: + """Total chunks in the store.""" + table = self.open_table(CHUNKS_TABLE) + return table.count_rows() if table is not None else 0 + def get_chunks_by_source(self, source: str) -> list[SearchChunk]: """Return every chunk whose ``source`` equals *source*.""" table = self.open_table(CHUNKS_TABLE) diff --git a/src/lilbee/data/store/fusion.py b/src/lilbee/data/store/fusion.py new file mode 100644 index 000000000..681fa2158 --- /dev/null +++ b/src/lilbee/data/store/fusion.py @@ -0,0 +1,77 @@ +"""Score-aware fusion of the vector and BM25 retrieval arms. + +Rank-reciprocal fusion (the previous mechanism) discards score magnitude by +construction: it cannot tell an arm that was certain from an arm that was +guessing, so one strong lexical hit for an identifier query drowns under +mediocre dense neighbors. Fusing normalized scores keeps that information: + + score = alpha * vector_similarity + (1 - alpha) * normalized_bm25 + +Vector similarity is ``1 - cosine_distance`` clamped to [0, 1], an absolute +signal. BM25 is unbounded and corpus-dependent, so it is normalized against +the top score of the result list (rank-stable, in (0, 1]). A row seen by only +one arm scores zero on the other, which is itself signal: lexical-only rows +survive fusion on their BM25 strength instead of being invisible to a +rank-based scheme fed from a shallow pool. +""" + +from __future__ import annotations + +from .types import SearchChunk + + +def vector_similarity(distance: float) -> float: + """Cosine distance to canonical [0, 1] similarity (distance spans [0, 2]).""" + return max(0.0, min(1.0, 1.0 - distance)) + + +def normalized_bm25(scores: list[float]) -> list[float]: + """Scale raw BM25 scores against the list maximum, into (0, 1]. + + BM25 has no absolute scale, so the top hit anchors the list; relative + strength within one query's results is the meaningful quantity. + Non-positive or absent maxima map everything to 0. + """ + top = max(scores, default=0.0) + if top <= 0.0: + return [0.0] * len(scores) + return [max(0.0, s) / top for s in scores] + + +def _key(chunk: SearchChunk) -> tuple[str, int]: + return (chunk.source, chunk.chunk_index) + + +def fuse_arms( + vector_rows: list[SearchChunk], + fts_rows: list[SearchChunk], + alpha: float, +) -> list[SearchChunk]: + """Merge the two arms into one list scored by convex combination. + + ``alpha`` weights the vector arm; ``1 - alpha`` the BM25 arm. Rows found + by both arms carry both provenance fields. The result is sorted by + ``score`` descending and deduplicated on ``(source, chunk_index)``. + """ + bm25_norms = dict( + zip( + (_key(r) for r in fts_rows), + normalized_bm25([r.bm25_score or 0.0 for r in fts_rows]), + strict=True, + ) + ) + merged: dict[tuple[str, int], SearchChunk] = {} + for row in vector_rows: + sim = vector_similarity(row.distance) if row.distance is not None else 0.0 + merged[_key(row)] = row.model_copy(update={"score": alpha * sim}) + for row in fts_rows: + key = _key(row) + lexical = (1.0 - alpha) * bm25_norms[key] + seen = merged.get(key) + if seen is None: + merged[key] = row.model_copy(update={"score": lexical}) + else: + merged[key] = seen.model_copy( + update={"score": (seen.score or 0.0) + lexical, "bm25_score": row.bm25_score} + ) + return sorted(merged.values(), key=lambda r: r.score or 0.0, reverse=True) diff --git a/src/lilbee/data/store/ranking.py b/src/lilbee/data/store/ranking.py index 5904269c1..cb2c1e58f 100644 --- a/src/lilbee/data/store/ranking.py +++ b/src/lilbee/data/store/ranking.py @@ -25,6 +25,7 @@ def mmr_rerank( results: list[SearchChunk], top_k: int, mmr_lambda: float | None = None, + relevance_scores: list[float] | None = None, ) -> list[SearchChunk]: """Maximal Marginal Relevance: select diverse results. Algorithm: Carbonell & Goldstein 1998, @@ -35,6 +36,12 @@ def mmr_rerank( 0.0 = maximum diversity, 1.0 = pure relevance. Defaults to ``cfg.mmr_lambda`` (0.5). + ``relevance_scores`` overrides the relevance term (one value per result, + higher = better). Fused hybrid results pass their canonical score here so + a lexically-certain hit with weak vector similarity keeps its standing; + without the override, relevance is cosine similarity to the query, which + would silently re-demote exactly the rows fusion promoted. + Complexity: O(top_k · N · D) time, O(N · D) space for N candidates of dimension D. Each outer iteration updates a running max-redundancy vector via one matmul rather than recomputing pairs pairwise. @@ -57,7 +64,10 @@ def mmr_rerank( query_norm = float(np.linalg.norm(query)) or 1.0 query_unit = query / query_norm - relevance = cand_unit @ query_unit # shape (N,) + if relevance_scores is not None: + relevance = np.asarray(relevance_scores, dtype=np.float32) + else: + relevance = cand_unit @ query_unit # shape (N,) n = len(results) max_redundancy = np.zeros(n, dtype=np.float32) diff --git a/src/lilbee/data/store/types.py b/src/lilbee/data/store/types.py index cfed71795..14963f945 100644 --- a/src/lilbee/data/store/types.py +++ b/src/lilbee/data/store/types.py @@ -114,9 +114,11 @@ def scope_to_chunk_type(scope: SearchScope | str | None) -> ChunkType | None: class SearchChunk(BaseModel): """A search result from LanceDB. - Hybrid results have ``relevance_score`` set (higher = better). - Vector-only results have ``distance`` set (lower = better). - Reranked results have ``rerank_score`` set (higher = better). + Every store search path sets ``score``: canonical [0, 1] relevance, + higher = better. Ranking, filtering, and selection compare only this + field; the arm-specific fields below it are provenance. + Vector-arm rows carry ``distance``; FTS-arm rows carry ``bm25_score``; + reranked rows additionally carry ``rerank_score`` (higher = better). """ model_config = ConfigDict(populate_by_name=True) @@ -139,14 +141,20 @@ def _coerce_none_chunk_type(cls, v: str | None) -> str: chunk_index: int vector: list[float] = Field(repr=False) distance: float | None = Field(None, alias="_distance") - # Hybrid rows carry an RRF ``_relevance_score`` (small fusion-scale magnitude, - # higher = better) that filtering and ranking compare across results. + # Legacy RRF ``_relevance_score`` (small fusion-scale magnitude, higher = + # better). Current store paths no longer populate it; it survives for rows + # produced by older code and external LanceDB rerankers. relevance_score: float | None = Field(None, validation_alias="_relevance_score") # FTS/BM25-only rows carry a raw, unbounded ``_score``. It lives in its own # field so it never contaminates the fusion-scale ``relevance_score``; only the # confidence-based expansion-skip reads it (sigmoid-squashed to [0, 1]). bm25_score: float | None = Field(None, validation_alias="_score") rerank_score: float | None = None + # Canonical relevance in [0, 1], set by the store on every search path: + # convex fusion of vector similarity and normalized BM25 on the hybrid + # path, clamped cosine similarity on vector-only, list-normalized BM25 on + # FTS-only probes. + score: float | None = None class SourceRecord(TypedDict): diff --git a/src/lilbee/retrieval/concepts/graph.py b/src/lilbee/retrieval/concepts/graph.py index f594f191d..289e8c1cc 100644 --- a/src/lilbee/retrieval/concepts/graph.py +++ b/src/lilbee/retrieval/concepts/graph.py @@ -177,7 +177,13 @@ def boost_results(self, results: list[Any], query_concepts: list[str]) -> list[A if overlap > 0: boost = (overlap / len(query_set)) * self._config.concept_boost_weight r = r.model_copy() - if r.relevance_score is not None: + if r.score is not None: + # Canonical [0, 1] space: the boost weight is directly + # comparable to an arm's fusion weight. (Added to a raw + # RRF score, whose whole range is ~0.017, the same 0.3 + # default swamped hybrid ranking outright.) + r.score = min(1.0, r.score + boost) + elif r.relevance_score is not None: r.relevance_score = r.relevance_score + boost elif r.distance is not None: r.distance = max(self._config.concept_boost_floor, r.distance - boost) diff --git a/src/lilbee/retrieval/query/dedup.py b/src/lilbee/retrieval/query/dedup.py index ee29e015a..f321b77a7 100644 --- a/src/lilbee/retrieval/query/dedup.py +++ b/src/lilbee/retrieval/query/dedup.py @@ -16,10 +16,13 @@ def _relevance_weight(result: SearchChunk) -> float: """Return a [0, 1] relevance weight for distance-aware selection. - Hybrid results (relevance_score set): use directly. - Vector results (distance set): invert cosine distance. - Neither: neutral default. + The canonical ``score`` is authoritative when present. Legacy rows fall + back to their arm-specific signal: RRF ``relevance_score`` (whose tiny + magnitude made hybrid rows weigh ~0.03 against ~0.4 for vector rows, + inverting the ranking inside greedy set cover) or inverted distance. """ + if result.score is not None: + return min(1.0, max(0.0, result.score)) if result.relevance_score is not None: return min(1.0, max(0.0, result.relevance_score)) if result.distance is not None: @@ -40,12 +43,14 @@ def normalize_scores(scores: list[float]) -> list[float]: def _fusion_signal(result: SearchChunk) -> float: """A chunk's retrieval confidence as a "higher = better" raw signal. - Hybrid rows carry an RRF ``relevance_score`` (small positive magnitude); - vector-only rows carry a cosine ``distance`` (0.0 = identical, lower = better). - ``is None`` rather than truthiness is deliberate: a perfect vector match has - ``distance == 0.0`` -- the strongest possible hit -- which falsy ``or`` would - misread as the neutral default. + The canonical ``score`` wins when present. Legacy rows: RRF + ``relevance_score`` (small positive magnitude) or cosine ``distance`` + (0.0 = identical, lower = better). ``is None`` rather than truthiness is + deliberate: a perfect vector match has ``distance == 0.0`` -- the + strongest possible hit -- which falsy ``or`` would misread as neutral. """ + if result.score is not None: + return result.score if result.relevance_score is not None: return result.relevance_score if result.distance is not None: @@ -56,16 +61,17 @@ def _fusion_signal(result: SearchChunk) -> float: def fusion_norms(results: list[SearchChunk]) -> list[float]: """Normalize each chunk's fusion signal to [0, 1] WITHIN its scoring family. - Hybrid rows carry an RRF ``relevance_score`` (tiny magnitude); the rest - (vector-only / HyDE recalls) carry a cosine ``distance``. The two scales are - not comparable, so normalizing them together would let one family dominate - purely as a scale artifact. Each family is scaled independently; a row with - neither signal sits in the non-RRF family at the neutral score. + Rows carrying the canonical ``score`` are one family (already mutually + comparable). Legacy rows split as before: RRF ``relevance_score`` (tiny + magnitude) versus cosine ``distance``; those scales are not comparable, + so normalizing them together would let one family dominate purely as a + scale artifact. A row with no signal at all sits at the neutral score. """ - rrf = [i for i, r in enumerate(results) if r.relevance_score is not None] - non_rrf = [i for i, r in enumerate(results) if r.relevance_score is None] + canonical = [i for i, r in enumerate(results) if r.score is not None] + rrf = [i for i, r in enumerate(results) if r.score is None and r.relevance_score is not None] + legacy = [i for i, r in enumerate(results) if r.score is None and r.relevance_score is None] norms = [_NEUTRAL_SCORE] * len(results) - for cohort in (rrf, non_rrf): + for cohort in (canonical, rrf, legacy): if not cohort: continue scaled = normalize_scores([_fusion_signal(results[i]) for i in cohort]) @@ -76,8 +82,9 @@ def fusion_norms(results: list[SearchChunk]) -> list[float]: def order_by_fusion(results: list[SearchChunk]) -> list[SearchChunk]: """Sort results best-first by fusion signal, normalized within each scoring - family so RRF (hybrid) and cosine-distance (vector/HyDE) rows are comparable - and one scale can't dominate the order as an artifact of its magnitude. + family so rows carrying different signals (canonical score, legacy RRF, + cosine distance) stay comparable and one scale can't dominate the order as + an artifact of its magnitude. """ norms = fusion_norms(results) order = sorted(range(len(results)), key=lambda i: norms[i], reverse=True) @@ -131,20 +138,30 @@ def filter_results( max_distance: float, min_relevance_score: float = 0.0, ) -> list[SearchChunk]: - """Drop results above max_distance or below min_relevance_score. - - Hybrid results (relevance_score set) are checked against min_relevance_score. - Vector results (distance set) are checked against max_distance. - Results with neither score pass through. When both scores are present, - relevance_score takes priority (hybrid results use RRF scoring, not - cosine distance). Pass max_distance=0 to disable distance filtering. + """Drop results below min_relevance_score or above max_distance. + + Rows carrying the canonical ``score`` are gated on ``min_relevance_score``, + which is what makes an abstention threshold possible: the score is [0, 1] + with real meaning, unlike RRF magnitudes. ``max_distance`` additionally + drops rows whose only signal is a far vector match (a row with lexical + support keeps its standing regardless of distance). Legacy rows keep the + old per-family checks. Pass max_distance=0 to disable distance filtering. """ if max_distance <= 0 and min_relevance_score <= 0: return results filtered: list[SearchChunk] = [] for r in results: - # Hybrid results: check relevance_score (takes priority over distance) - if r.relevance_score is not None: + if r.score is not None: + if min_relevance_score > 0 and r.score < min_relevance_score: + continue + if ( + max_distance > 0 + and r.bm25_score is None + and r.distance is not None + and r.distance > max_distance + ): + continue + elif r.relevance_score is not None: if min_relevance_score > 0 and r.relevance_score < min_relevance_score: continue elif r.distance is not None and max_distance > 0 and r.distance > max_distance: @@ -173,7 +190,15 @@ def deduplicate_sources( def _sort_key(r: SearchChunk) -> float: - """Sort key: lower = more relevant.""" + """Sort key: lower = more relevant. + + Canonical ``score`` first. The legacy branches are ordered so that + comparing a leftover RRF row (key ~ -0.03) with a distance row (key >= 0) + keeps the old bias only among rows the store did not score; scored rows + never hit them. + """ + if r.score is not None: + return -r.score if r.relevance_score is not None: return -r.relevance_score if r.distance is not None: diff --git a/src/lilbee/retrieval/query/expansion.py b/src/lilbee/retrieval/query/expansion.py index b986fe51a..ba03126b3 100644 --- a/src/lilbee/retrieval/query/expansion.py +++ b/src/lilbee/retrieval/query/expansion.py @@ -1,4 +1,4 @@ -"""Constants for LLM-driven query expansion.""" +"""Constants for LLM-driven query expansion and history condensation.""" from __future__ import annotations @@ -9,3 +9,16 @@ ) EXPANSION_MAX_TOKENS = 200 + +CONDENSE_PROMPT = ( + "Rewrite the follow-up question as one standalone search query, resolving " + "pronouns and references using the conversation. Return ONLY the rewritten " + "query, nothing else. If the question already stands alone, return it " + "unchanged.\n\nConversation:\n{history}\n\nFollow-up question: {question}" +) + +CONDENSE_MAX_TOKENS = 120 + +# History turns included in the condensation prompt; older turns rarely change +# what a follow-up refers to and only add latency. +CONDENSE_HISTORY_TURNS = 6 diff --git a/src/lilbee/retrieval/query/formatting.py b/src/lilbee/retrieval/query/formatting.py index bf3b3dac2..8bcbfc72d 100644 --- a/src/lilbee/retrieval/query/formatting.py +++ b/src/lilbee/retrieval/query/formatting.py @@ -14,7 +14,13 @@ Question: {question}""" -_CITE_REF_RE = re.compile(r"\[(\d+)\]") +# Bracketed citation groups: [1], [1, 2], [1-3], [1, 3-5]. Models mix all of +# these despite being asked for single [n] markers; matching only [n] made +# cited_sources under-count and fed JSON consumers false-negative grounding. +_CITE_GROUP_RE = re.compile(r"\[(\d+(?:\s*[-,]\s*\d+)*)\]") +_CITE_RANGE_RE = re.compile(r"(\d+)\s*-\s*(\d+)") +# Ranges wider than this are page spans or line numbers, not citation lists. +_MAX_CITE_RANGE = 32 # Matches trailing LLM-generated citation blocks like "Key sources:", "Sources:", # "References:", "Bibliography:", "Citations:" (with optional markdown heading). @@ -61,6 +67,17 @@ def _format_citation(citation: CitationRecord) -> str: return f" → {source_display}" +def _location_suffix(result: SearchChunk) -> str: + """The page or line span of a chunk, or empty when neither applies.""" + if result.content_type == "pdf": + ps, pe = result.page_start, result.page_end + return f"page {ps}" if ps == pe else f"pages {ps}-{pe}" + if result.content_type == "code": + ls, le = result.line_start, result.line_end + return f"line {ls}" if ls == le else f"lines {ls}-{le}" + return "" + + def format_source(result: SearchChunk, citations: list[CitationRecord] | None = None) -> str: """Format a search result as a source citation line. For wiki chunks, shows the wiki page path followed by indented transitive citations. @@ -72,33 +89,73 @@ def format_source(result: SearchChunk, citations: list[CitationRecord] | None = parts.append(_format_citation(cit)) return "\n".join(parts) - if result.content_type == "pdf": - ps, pe = result.page_start, result.page_end - pages = f"page {ps}" if ps == pe else f"pages {ps}-{pe}" - return f" → {source_display}, {pages}" + location = _location_suffix(result) + if location: + return f" → {source_display}, {location}" + return f" → {source_display}" - if result.content_type == "code": - ls, le = result.line_start, result.line_end - lines = f"line {ls}" if ls == le else f"lines {ls}-{le}" - return f" → {source_display}, {lines}" - return f" → {source_display}" +def _context_header(result: SearchChunk) -> str: + """One-line provenance for a context block: source name plus location. + + Without it the answering model sees bare numbered text: it cannot + attribute a claim to a named document, notice two chunks share a source, + or confirm it is reading the document the user asked about. + """ + location = _location_suffix(result) + if location: + return f"{result.source}, {location}" + return result.source def build_context(results: list[SearchChunk]) -> str: - """Build context block from search results.""" - return "\n\n".join(f"[{i}] {r.chunk}" for i, r in enumerate(results, 1)) + """Build context block from search results, one provenance header each.""" + return "\n\n".join(f"[{i}] ({_context_header(r)})\n{r.chunk}" for i, r in enumerate(results, 1)) def _extract_cited_indices(text: str) -> set[int]: - """Extract [N] citation references from LLM answer text.""" - return {int(m.group(1)) for m in _CITE_REF_RE.finditer(text)} + """Extract citation references from LLM answer text: [1], [1, 2], [1-3].""" + indices: set[int] = set() + for m in _CITE_GROUP_RE.finditer(text): + group = m.group(1) + remainder = _CITE_RANGE_RE.sub("", group) + for start, end in _CITE_RANGE_RE.findall(group): + lo, hi = int(start), int(end) + if lo <= hi <= lo + _MAX_CITE_RANGE: + indices.update(range(lo, hi + 1)) + indices.update(int(n) for n in re.findall(r"\d+", remainder)) + return indices + + +def _identifier_shaped(stem: str) -> bool: + """Whether a filename stem is distinctive enough to match in prose. + + A stem carrying a digit or a separator ("survey_report", "ARC-00000482") + only appears in an answer when the model names the document; a bare word + stem ("notes") collides with ordinary prose and cannot be trusted. + """ + return any(c.isdigit() or c in "_-" for c in stem) def cited_subset(answer: str, sources: list[SearchChunk]) -> list[SearchChunk]: - """The sources the answer actually cited via [n] markers, in order (empty if none).""" + """The sources the answer actually referenced, in order (empty if none). + + ``[n]`` markers are the primary signal. Name mentions count too: context + blocks show the model each source's name, and models often attribute by + name ("according to survey_report.pdf") instead of by marker, which + previously read as an ungrounded answer to JSON consumers. + """ cited = _extract_cited_indices(answer) - return [sources[i - 1] for i in sorted(cited) if 1 <= i <= len(sources)] + picked = {i - 1 for i in cited if 1 <= i <= len(sources)} + lowered = answer.lower() + for i, source in enumerate(sources): + if i in picked: + continue + name = Path(source.source).name.lower() + stem = Path(source.source).stem + if name in lowered or (_identifier_shaped(stem) and stem.lower() in lowered): + picked.add(i) + return [sources[i] for i in sorted(picked)] def strip_llm_citations(text: str) -> str: diff --git a/src/lilbee/retrieval/query/intent.py b/src/lilbee/retrieval/query/intent.py new file mode 100644 index 000000000..9d0e202ea --- /dev/null +++ b/src/lilbee/retrieval/query/intent.py @@ -0,0 +1,157 @@ +"""Query intent detection: known-item lookups and corpus aggregates. + +Top-k similarity retrieval answers topical questions. Two other question +shapes reach the same pipe and fail structurally: + +- A known-item lookup ("summarize survey_214.pdf") names the thing it + wants; the answer is a document, not a ranking. +- An aggregate ("how many documents mention the observatory") is a property + of the whole corpus; the top 20 of 500k chunks cannot count anything. + +Detection here is deterministic and deliberately conservative: a missed +route degrades to topical retrieval, which handles the query the way it +always has, while a false positive would hijack a topical question. Every +pattern therefore requires an explicit structural cue. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + + +class AggregateKind(Enum): + """What a count-shaped question is asking to count.""" + + TOTAL_SOURCES = "total_sources" + TERM_MENTIONS = "term_mentions" + UNSUPPORTED = "unsupported" + + +@dataclass(frozen=True) +class AggregateQuery: + """A parsed aggregate question.""" + + kind: AggregateKind + term: str = "" + + +# Filename-shaped tokens: a path-ish word with a known document extension. +# No spaces: a name containing them arrives quoted and the quote pattern +# catches it; allowing spaces here would swallow leading sentence words. +_FILENAME_RE = re.compile( + r"[\w.-][\w./-]*\.(?:pdf|md|txt|docx?|rst|html?|epub|csv|py|rs|js|ts|java|go)\b", + re.IGNORECASE, +) + +# "document 214", "doc #47", "exhibit 12b", "file 2020-03": a document noun +# followed by a short identifier. +_DOC_REF_RE = re.compile( + r"\b(?:document|doc|file|exhibit|attachment|report)\s+#?([\w][\w.-]{0,23})\b", + re.IGNORECASE, +) + +# Quoted names: 'harbor survey 2002' / "harbor survey 2002". +_QUOTED_RE = re.compile(r"[\"']([^\"']{2,80})[\"']") + +# Generic nouns that follow a document noun in topical questions ("the +# documents mention...", "which document says..."); never identifiers. +_REF_STOPWORDS = frozenset( + {"that", "which", "the", "this", "these", "those", "it", "them", "was", "is", "are"} +) + +_HOW_MANY_RE = re.compile( + r"^\s*(?:roughly\s+|approximately\s+|about\s+)?how\s+many\b", re.IGNORECASE +) + +# "how many documents/sources/files are there/indexed": corpus totals. +_TOTAL_RE = re.compile( + r"how\s+many\s+(?:documents|sources|files|pages|chunks)\s*" + r"(?:are\s+(?:there|indexed|in\s+the\s+index)|do(?:es)?\s+.*\b(?:index|corpus|vault)\b.*)?[?\s]*$", + re.IGNORECASE, +) + +# "how many documents mention/contain/reference X": term-mention counts. +_TERM_MENTION_RE = re.compile( + r"how\s+many\s+(?:documents|sources|files|pages|chunks|passages)\s+" + r"(?:mention|mentions|mentioning|contain|contains|containing|reference|references|referencing|discuss|discussing)\s+" + r"(.+?)[?.\s]*$", + re.IGNORECASE, +) + + +def document_references(question: str) -> list[str]: + """Candidate document identifiers named in *question*, best-first. + + Filenames beat quoted names beat "document N" references; all are + resolved against real source metadata by the caller, so a wrong + candidate costs one lookup, not a wrong route. + """ + candidates: list[str] = [] + for m in _FILENAME_RE.finditer(question): + candidates.append(m.group(0).strip()) + for m in _QUOTED_RE.finditer(question): + candidates.append(m.group(1).strip()) + for m in _DOC_REF_RE.finditer(question): + ref = m.group(1).strip() + if ref.lower() not in _REF_STOPWORDS: + candidates.append(ref) + seen: set[str] = set() + unique = [] + for c in candidates: + key = c.lower() + if key not in seen: + seen.add(key) + unique.append(c) + return unique + + +_TOKEN_SPLIT_RE = re.compile(r"[^0-9A-Za-z]+") + + +def matches_reference(ref: str, filename: str) -> bool: + """Whether *filename* names the document *ref* refers to, token-exactly. + + Substring search cannot resolve a bare number against zero-padded ids: + "482" is a substring of both "...00000482" and "...00010482". Tokens + split on non-alphanumerics are compared whole; numeric tokens compare by + value so leading zeros don't hide the match, and a longer number sharing + a suffix stays a non-match. + """ + ref_token = ref.strip().lower() + if ref_token in (filename.lower(), Path(filename).name.lower()): + return True + stem = Path(filename).stem.lower() + for token in _TOKEN_SPLIT_RE.split(stem): + if not token: + continue + if token == ref_token: + return True + if token.isdigit() and ref_token.isdigit() and int(token) == int(ref_token): + return True + return False + + +def parse_aggregate(question: str) -> AggregateQuery | None: + """Parse a count-shaped question, or ``None`` for anything else. + + Only "how many ..." questions qualify; of those, term-mention and + corpus-total forms are answerable against today's schema. The rest + (counts over typed records the store does not hold) come back as + ``UNSUPPORTED`` so the caller can decline precisely instead of feeding + the question to top-k retrieval that structurally cannot count. + """ + if not _HOW_MANY_RE.search(question): + return None + m = _TERM_MENTION_RE.search(question) + if m: + term = m.group(1).strip().strip("\"'") + # Strip a leading article so 'mention the observatory' counts 'observatory'. + term = re.sub(r"^(?:the|a|an)\s+", "", term, flags=re.IGNORECASE) + if term: + return AggregateQuery(AggregateKind.TERM_MENTIONS, term=term) + if _TOTAL_RE.search(question): + return AggregateQuery(AggregateKind.TOTAL_SOURCES) + return AggregateQuery(AggregateKind.UNSUPPORTED) diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index 78225e08d..88bc6e6a1 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -import math +import re from collections.abc import Generator from datetime import datetime from typing import TYPE_CHECKING, Any, cast @@ -32,7 +32,13 @@ order_by_fusion, prepare_results, ) -from lilbee.retrieval.query.expansion import EXPANSION_MAX_TOKENS, EXPANSION_PROMPT +from lilbee.retrieval.query.expansion import ( + CONDENSE_HISTORY_TURNS, + CONDENSE_MAX_TOKENS, + CONDENSE_PROMPT, + EXPANSION_MAX_TOKENS, + EXPANSION_PROMPT, +) from lilbee.retrieval.query.formatting import ( CONTEXT_TEMPLATE, build_context, @@ -40,6 +46,13 @@ strip_llm_citations, ) from lilbee.retrieval.query.history_window import estimate_text_tokens +from lilbee.retrieval.query.intent import ( + AggregateKind, + AggregateQuery, + document_references, + matches_reference, + parse_aggregate, +) from lilbee.retrieval.query.memory import format_memory_block from lilbee.retrieval.query.tokenize import _idf_weights, _tokenize from lilbee.retrieval.reasoning import ( @@ -60,21 +73,43 @@ # scores for the expansion-skip heuristic. _MIN_BM25_PROBE_RESULTS = 2 +# Substring candidates fetched per document reference before token-exact +# disambiguation picks the unique winner. +_KNOWN_ITEM_CANDIDATES = 50 + # Structured-query mode names (the ``mode:`` prefix shortcut). Single source for # both the prefix parser and the dispatch in ``_search_structured``. "term"/"vec"/ # "hyde" pick a retrieval strategy; "wiki"/"raw" are ChunkType scope shortcuts. _STRUCTURED_QUERY_MODES = ("term", "vec", "hyde", ChunkType.WIKI.value, ChunkType.RAW.value) +# Leading list markers models prepend to expansion output despite the prompt: +# "1.", "2)", "-", "*", "•". +_LIST_MARKER_RE = re.compile(r"^\s*(?:\d+[.)]\s*|[-*•]\s+)") + + +def _strip_list_marker(line: str) -> str: + """Drop a leading list marker from an expansion variant line.""" + return _LIST_MARKER_RE.sub("", line).strip() + + +# Half-saturation constant for BM25 confidence: a raw score of 5 reads as 0.5, +# 20 as 0.8, 45 as 0.9. A plain sigmoid saturated at ~0.99 for any raw score +# above 5, which made the top-vs-runner-up gap condition unsatisfiable exactly +# when BM25 was most certain, so the skip never fired. +_BM25_HALF_SATURATION = 5.0 + def _bm25_confidence(score: float | None) -> float: - """Squash a raw, unbounded BM25 score into the (0, 1) confidence that the - expansion-skip thresholds are calibrated for (the config calls this the - "sigmoid-normalized BM25 score"). Absent or non-positive scores read as 0, - so a missing FTS signal never trips the skip. + """Squash a raw, unbounded BM25 score into (0, 1) without saturating. + + ``s / (s + k)`` keeps strong scores distinguishable (unlike a sigmoid, + which flattens everything past ~5 to within 0.01 of 1.0). Absent or + non-positive scores read as 0, so a missing FTS signal never trips the + expansion skip. """ if score is None or score <= 0.0: return 0.0 - return 1.0 / (1.0 + math.exp(-score)) + return score / (score + _BM25_HALF_SATURATION) # RAG mode answer when retrieval finds no usable sources: a grounded refusal @@ -94,8 +129,9 @@ def _bm25_confidence(score: float | None) -> float: _ANSWER_RESERVE_TOKENS = 1024 # Approximate token cost of the Context/Question template wrapper. _CONTEXT_TEMPLATE_TOKENS = 16 -# Approximate per-source overhead: the "[i] " marker plus blank-line separator. -_PER_SOURCE_TOKENS = 8 +# Approximate per-source overhead: the "[i] " marker, the provenance header +# (source path plus page/line span), and the blank-line separator. +_PER_SOURCE_TOKENS = 24 class ChatMessage(TypedDict): @@ -189,15 +225,22 @@ def _concept_query_expansion(self, question: str) -> list[str]: return [] def _llm_expand(self, question: str, count: int) -> list[str]: - """Call the LLM to produce ``count`` alternative phrasings.""" + """Call the LLM to produce ``count`` alternative phrasings. + + Reasoning is stripped before the line split: a reasoning chat model + otherwise contributes its deliberation as "variants" that get embedded + and searched. List numbering is stripped per line since models add it + despite the prompt, and "1." pollutes the BM25 arm of every variant + search. + """ prompt = EXPANSION_PROMPT.format(count=count, question=question) messages = [{"role": "user", "content": prompt}] response = self._provider.chat( messages, stream=False, options={"num_predict": EXPANSION_MAX_TOKENS} ) - text = response.text.strip() - variants = [line.strip() for line in text.split("\n") if line.strip()] - return variants[:count] + text = strip_reasoning(response.text).strip() + variants = [_strip_list_marker(line.strip()) for line in text.split("\n") if line.strip()] + return [v for v in variants if v][:count] def _expand_query( self, question: str, question_vec: list[float] @@ -246,13 +289,17 @@ def _should_skip_expansion(self, question: str, chunk_type: ChunkType | None = N ) if not results: return False - top_score = _bm25_confidence(results[0].bm25_score) - if top_score < self._config.expansion_skip_threshold: + top_raw = results[0].bm25_score or 0.0 + if _bm25_confidence(top_raw) < self._config.expansion_skip_threshold: return False if len(results) < _MIN_BM25_PROBE_RESULTS: return True - second_score = _bm25_confidence(results[1].bm25_score) - return (top_score - second_score) >= self._config.expansion_skip_gap + # Relative gap in raw score space: any squash compresses the spread + # between two strong scores toward zero, so a squashed-space gap test + # can never fire exactly when the lexical arm is most certain. + second_raw = results[1].bm25_score or 0.0 + relative_gap = (top_raw - second_raw) / top_raw if top_raw > 0 else 0.0 + return relative_gap >= self._config.expansion_skip_gap def _apply_concept_boost(self, results: list[SearchChunk], question: str) -> list[SearchChunk]: if not self._config.concept_graph or not results: @@ -294,7 +341,9 @@ def _hyde_search( stream=False, options={"num_predict": EXPANSION_MAX_TOKENS}, ) - text = response.text.strip() + # Reasoning models front-load deliberation; embedding it instead + # of the passage would search for the model's thought process. + text = strip_reasoning(response.text).strip() if not text: return [] hyde_vec = self._embedder.embed_query(text) @@ -413,13 +462,15 @@ def _merge_hyde_results( top_k: int, chunk_type: ChunkType | None = None, ) -> None: - """Append unseen HyDE hits to ``results`` (in place), reweighted by ``hyde_weight``.""" + """Append unseen HyDE hits to ``results`` (in place), down-weighted by + ``hyde_weight`` in canonical score space (a weight of 1.0 trusts HyDE + hits as much as direct hits; lower discounts them proportionally).""" for r in self._hyde_search(question, top_k, chunk_type=chunk_type): key = (r.source, r.chunk_index) if key in seen: continue - if r.distance is not None and self._config.hyde_weight > 0: - r = r.model_copy(update={"distance": r.distance / self._config.hyde_weight}) + if r.score is not None: + r = r.model_copy(update={"score": r.score * self._config.hyde_weight}) results.append(r) seen.add(key) @@ -472,11 +523,77 @@ def search( if self._config.hyde: self._merge_hyde_results(question, results, seen, top_k, chunk_type) results = self._apply_concept_boost(results, question) + # Merged variant/HyDE hits arrive appended, not ranked; every consumer + # of this method (bare search surfaces included) gets one global order + # over the canonical score rather than insertion order. + results = order_by_fusion(results) # Apply the date-range filter here so the bare search() path (e.g. /api/search) # honors a "recent"/"today" query, matching the chat/ask path. results = self._apply_temporal_filter(results, question) return results[: top_k * 2] + def _condense_question(self, question: str, history: list[ChatMessage]) -> str: + """Rewrite a follow-up into a standalone retrieval query. + + Retrieval sees only the query text; without this, "what about his + brother?" is embedded and BM25-matched with its pronouns. The + rewritten form drives retrieval only; the user's original wording + still reaches the answering prompt. Falls back to the original + question on any failure or empty rewrite. + """ + recent = history[-CONDENSE_HISTORY_TURNS:] + transcript = "\n".join(f"{m['role']}: {m['content']}" for m in recent) + prompt = CONDENSE_PROMPT.format(history=transcript, question=question) + try: + response = self._provider.chat( + [{"role": "user", "content": prompt}], + stream=False, + options={"num_predict": CONDENSE_MAX_TOKENS}, + ) + rewritten = strip_reasoning(response.text).strip().splitlines() + first_line = rewritten[0].strip() if rewritten else "" + if first_line: + log.debug("Condensed follow-up %r -> %r", question, first_line) + return first_line + except Exception: + log.debug("History condensation failed; using the raw question", exc_info=True) + return question + + def _known_item_results(self, question: str) -> list[SearchChunk]: + """Resolve a document named in *question* to its own chunks. + + A question that names a document wants that document, not a ranking: + similarity search retrieves neighbors of the question's wording, + which for "summarize survey_214.pdf" is mostly noise. Resolution is + conservative: only a reference matching exactly one source routes; + anything ambiguous falls back to topical retrieval. Chunks come back + in document order with full canonical confidence, since their + relevance is established by the name match, not by similarity. + """ + for ref in document_references(question): + candidates = self._store.get_sources(search=ref, limit=_KNOWN_ITEM_CANDIDATES) + # Substring search over-matches (a bare "482" hits every + # zero-padded id containing it); token-exact matching does the + # disambiguation, and only a unique winner routes. + matches = [s for s in candidates if matches_reference(ref, s["filename"])] + if len(matches) != 1: + # A unique substring hit still routes for word refs (quoted + # titles never token-match hyphenated filenames), but not for + # numeric ones: "12" inside "notes-2012" is the exact false + # match token comparison exists to reject. + if len(candidates) == 1 and not ref.strip().isdigit(): + matches = candidates + else: + continue + filename = matches[0]["filename"] + chunks = self._store.get_chunks_by_source(filename) + if not chunks: + continue + chunks.sort(key=lambda c: c.chunk_index) + log.info("Known-item route: %r resolved to %s", ref, filename) + return [c.model_copy(update={"score": 1.0}) for c in chunks] + return [] + def build_rag_context( self, question: str, @@ -490,17 +607,29 @@ def build_rag_context( ``chunk_type`` restricts the pool to ``"raw"`` or ``"wiki"`` rows; ``None`` (default) searches the mixed pool. """ - results = self.search(question, top_k=top_k, chunk_type=chunk_type) - results = filter_results( - results, self._config.max_distance, self._config.min_relevance_score + retrieval_query = question + if history and self._config.history_rewrite: + retrieval_query = self._condense_question(question, history) + known_item = ( + self._known_item_results(retrieval_query) if self._config.intent_routing else [] ) - if not results: - return None - results = prepare_results(results) - if self._config.reranker_model: - results = self._reranker.rerank(question, results) - # Temporal filtering already ran inside search(); no need to repeat it here. - results = self.select_context(results, question) + if known_item: + # The named document IS the context; ranking and reranking would + # only reorder or drop parts of it. The budget fit below still + # trims to the context window, keeping the document's head. + results = known_item + else: + results = self.search(retrieval_query, top_k=top_k, chunk_type=chunk_type) + results = filter_results( + results, self._config.max_distance, self._config.min_relevance_score + ) + if not results: + return None + results = prepare_results(results) + if self._config.reranker_model: + results = self._reranker.rerank(retrieval_query, results) + # Temporal filtering already ran inside search(); no need to repeat it here. + results = self.select_context(results, retrieval_query) system = self._system_with_memory(self._config.rag_system_prompt, question) results = self._fit_context_budget(results, system, question, history) context = build_context(results) @@ -580,6 +709,50 @@ def _memory_block(self, question: str) -> str: ) return format_memory_block(preferences, facts, self._config.memory_token_budget) + def _answer_aggregate(self, aggregate: AggregateQuery) -> str: + """Answer a count-shaped question with an exact full-corpus scan. + + Top-k retrieval sees a handful of chunks out of the whole corpus, so + it structurally cannot count; the faithful-but-useless outcome is a + model hedging that "the context does not provide precise counts". + Counting is a scan, and a scan needs no language model: the numbers + below are exact, not generated. + """ + if aggregate.kind is AggregateKind.TOTAL_SOURCES: + sources = self._store.count_sources() + chunks = self._store.count_chunks() + return f"The index holds {sources} documents split into {chunks} searchable passages." + if aggregate.kind is AggregateKind.TERM_MENTIONS: + chunk_hits, source_hits = self._store.count_term_mentions(aggregate.term) + return ( + f"Exact scan of the whole index: {source_hits} documents mention " + f"{aggregate.term!r}, across {chunk_hits} passages. This counts literal " + f"mentions of the phrase, not paraphrases." + ) + return ( + "Answering that count needs structured records (dates, identifiers, or " + "entities) that aren't extracted from this corpus yet. I can count " + "documents or passages that mention a specific term, or you can ask for " + "the passages themselves and count from those." + ) + + def route_direct_answer(self, question: str) -> str | None: + """The exact-scan answer for a count-shaped question, else ``None``. + + Every retrieval entry point must consult this before building RAG + context: ask_raw/ask_stream do (covering CLI and TUI), and the HTTP + handlers call it themselves because they assemble their own prompts + from build_rag_context. An entry point that skips it hedges at the + count questions every other surface answers exactly. + """ + if not self._config.intent_routing: + return None + aggregate = parse_aggregate(question) + if aggregate is None: + return None + log.info("Aggregate route: %s for %r", aggregate.kind.value, question) + return self._answer_aggregate(aggregate) + def skip_retrieval(self) -> bool: """Whether this turn should bypass RAG: chat-only mode or no embedder.""" return ( @@ -647,6 +820,9 @@ def ask_raw( return AskResult(answer=SEARCH_NEEDS_EMBEDDER, sources=[]) if self.skip_retrieval(): return AskResult(answer=self._direct_chat(question, history, options), sources=[]) + aggregate_answer = self.route_direct_answer(question) + if aggregate_answer is not None: + return AskResult(answer=aggregate_answer, sources=[]) rag = self.build_rag_context(question, top_k=top_k, history=history, chunk_type=chunk_type) if rag is None: return AskResult(answer=_GROUNDED_REFUSAL, sources=[]) @@ -717,6 +893,10 @@ def ask_stream( if self.skip_retrieval(): yield from self._stream_direct(question, history, options) return + aggregate_answer = self.route_direct_answer(question) + if aggregate_answer is not None: + yield StreamToken(content=aggregate_answer, is_reasoning=False) + return rag = self.build_rag_context(question, top_k=top_k, history=history, chunk_type=chunk_type) if rag is None: diff --git a/src/lilbee/server/handlers/rag.py b/src/lilbee/server/handlers/rag.py index a5fe3639a..abb233101 100644 --- a/src/lilbee/server/handlers/rag.py +++ b/src/lilbee/server/handlers/rag.py @@ -248,24 +248,18 @@ async def _stream_rag_response( yield sse_event(SseEvent.SOURCES, []) yield sse_done({}) return - if searcher.skip_retrieval(): - # Chat mode with an embedder present: answer ungrounded, mirroring ask_raw. - results: list[SearchChunk] = [] - messages = searcher.direct_messages(question, history) - else: - try: - rag = searcher.build_rag_context( - question, top_k=top_k, history=history, chunk_type=chunk_type - ) - except EmbeddingModelMismatchError as mismatch: - # detail carries the index's embedder so the client can offer to adopt it. - detail = mismatch.persisted_model if mismatch.dims_match else None - yield sse_error(str(mismatch), code=SseErrorCode.INDEX_EMBEDDER_MISMATCH, detail=detail) - return - if rag is None: - yield sse_error("No relevant documents found.") - return - results, messages = rag + results, messages, preempt = _resolve_stream_context( + searcher, + question, + history, + top_k, + chunk_type, + retrieval_off=searcher.skip_retrieval(), + ) + for frame in preempt: + yield frame + if messages is None: + return opts = _resolve_generation_options(options) or cfg.generation_options() @@ -336,6 +330,11 @@ async def chat( # Search mode with no embedder can't ground; refuse cleanly with the same # message ask returns instead of silently answering off-corpus. return AskResponse(answer=SEARCH_NEEDS_EMBEDDER, sources=[], cited_sources=[]) + if not _retrieval_off(searcher, top_k): + # Count-shaped questions get the exact-scan answer, mirroring ask_raw. + direct = searcher.route_direct_answer(question) + if direct is not None: + return AskResponse(answer=direct, sources=[], cited_sources=[]) sources, messages = _build_chat_messages(question, history, top_k, chunk_type) req = _build_canonical_request(messages, options) response = await asyncio.to_thread(dispatch_chat, req) @@ -389,25 +388,18 @@ async def _stream_chat_response( yield sse_event(SseEvent.SOURCES, []) yield sse_done({}) return - if _retrieval_off(searcher, top_k): - # Chat-only mode, no embedder, or an explicit top_k:0 pure-LLM call: - # answer directly with no RAG context. - sources: list[SearchChunk] = [] - messages = searcher.direct_messages(question, history) - else: - try: - rag = searcher.build_rag_context( - question, top_k=top_k or 0, history=history, chunk_type=chunk_type - ) - except EmbeddingModelMismatchError as exc: - # detail carries the index's embedder so the client can offer to adopt it. - detail = exc.persisted_model if exc.dims_match else None - yield sse_error(str(exc), code=SseErrorCode.INDEX_EMBEDDER_MISMATCH, detail=detail) - return - if rag is None: - yield sse_error("No relevant documents found.") - return - sources, messages = rag + sources, messages, preempt = _resolve_stream_context( + searcher, + question, + history, + top_k, + chunk_type, + retrieval_off=_retrieval_off(searcher, top_k), + ) + for frame in preempt: + yield frame + if messages is None: + return req = _build_canonical_request(messages, options) answer_parts: list[str] = [] @@ -550,6 +542,48 @@ def _retrieval_off(searcher: Searcher, top_k: int | None) -> bool: return top_k == 0 or searcher.skip_retrieval() +def _resolve_stream_context( + searcher: Searcher, + question: str, + history: list[ChatMessage] | None, + top_k: int | None, + chunk_type: ChunkType | None, + *, + retrieval_off: bool, +) -> tuple[list[SearchChunk], list[ChatMessage] | None, list[str]]: + """Resolve retrieval for a streaming handler: (sources, messages, preempt). + + Non-empty ``preempt`` frames are emitted verbatim; ``messages`` of ``None`` + means the stream ends after them (a direct exact-scan answer or an error). + Shared by the ask and chat streams so the two paths cannot drift: both + route count questions to the exact scan, surface an embedder mismatch as + a coded SSE error, and report empty retrieval the same way. + """ + if retrieval_off: + return [], searcher.direct_messages(question, history), [] + direct = searcher.route_direct_answer(question) + if direct is not None: + frames = [ + sse_event(SseEvent.TOKEN, {"token": direct}), + sse_event(SseEvent.SOURCES, []), + sse_done({}), + ] + return [], None, frames + try: + rag = searcher.build_rag_context( + question, top_k=top_k or 0, history=history, chunk_type=chunk_type + ) + except EmbeddingModelMismatchError as mismatch: + # detail carries the index's embedder so the client can offer to adopt it. + detail = mismatch.persisted_model if mismatch.dims_match else None + frame = sse_error(str(mismatch), code=SseErrorCode.INDEX_EMBEDDER_MISMATCH, detail=detail) + return [], None, [frame] + if rag is None: + return [], None, [sse_error("No relevant documents found.")] + results, messages = rag + return results, messages, [] + + def _build_chat_messages( question: str, history: list[ChatMessage], diff --git a/src/lilbee/server/models.py b/src/lilbee/server/models.py index fc9c54b6d..9ce7368ca 100644 --- a/src/lilbee/server/models.py +++ b/src/lilbee/server/models.py @@ -123,6 +123,9 @@ class CleanedChunk(BaseModel): distance: float | None = None relevance_score: float | None = None rerank_score: float | None = None + # Canonical [0, 1] relevance from score fusion; the ranking signal HTTP + # clients should sort and threshold on (relevance_score is legacy RRF). + score: float | None = None page_start: int = 0 page_end: int = 0 line_start: int = 0 diff --git a/tests/test_concepts.py b/tests/test_concepts.py index 4866460e9..2aaf8a7a4 100644 --- a/tests/test_concepts.py +++ b/tests/test_concepts.py @@ -97,6 +97,7 @@ def _make_result( chunk="some text", distance=0.5, relevance_score=None, + score=None, ) -> SearchChunk: return SearchChunk( source=source, @@ -109,6 +110,7 @@ def _make_result( chunk_index=chunk_index, distance=distance, relevance_score=relevance_score, + score=score, vector=[0.1], ) @@ -509,6 +511,21 @@ def test_boost_results_relevance_score(self, cg, mock_svc): boosted = cg.boost_results(results, ["python"]) assert boosted[0].relevance_score > 0.8 + def test_boost_canonical_score_bounded(self, cg, mock_svc): + """Boost applies in canonical [0, 1] score space and clamps at 1.0.""" + results = [ + _make_result(chunk_index=0, score=0.5), + _make_result(chunk_index=1, score=0.95), + ] + mock_table = MagicMock() + mock_table.search.return_value.where.return_value.to_list.return_value = [ + {"concept": "python"}, + ] + mock_svc.store.open_table.return_value = mock_table + boosted = cg.boost_results(results, ["python"]) + assert boosted[0].score == pytest.approx(0.5 + cfg.concept_boost_weight) + assert boosted[1].score == 1.0 + def test_boost_results_no_overlap(self, cg, mock_svc): results = [_make_result(distance=0.5, chunk_index=0)] mock_table = MagicMock() diff --git a/tests/test_intent.py b/tests/test_intent.py new file mode 100644 index 000000000..f66c5b0d6 --- /dev/null +++ b/tests/test_intent.py @@ -0,0 +1,139 @@ +"""Tests for query intent detection and routing.""" + +import pytest + +from lilbee.retrieval.query.intent import ( + AggregateKind, + document_references, + matches_reference, + parse_aggregate, +) + + +class TestDocumentReferences: + def test_filename_reference(self): + assert document_references("summarize survey_report.pdf for me") == ["survey_report.pdf"] + + def test_document_number_reference(self): + assert document_references("summarize Document 214") == ["214"] + + def test_quoted_reference(self): + assert document_references('what does "harbor survey 2010" say') == ["harbor survey 2010"] + + def test_filename_beats_doc_number(self): + refs = document_references("compare report 12 with summary_final.md") + assert refs[0] == "summary_final.md" + + def test_topical_question_yields_nothing(self): + assert document_references("which documents mention the harbor?") == [] + assert document_references("what do the files say about travel?") == [] + + def test_generic_noun_after_document_is_not_a_reference(self): + assert document_references("is there a document that lists the deliveries?") == [] + + +class TestMatchesReference: + def test_numeric_ref_matches_zero_padded_token(self): + assert matches_reference("482", "ARC-REC-00000482.pdf") + + def test_numeric_ref_rejects_different_number_with_shared_suffix(self): + assert not matches_reference("482", "ARC-REC-00010482.pdf") + + def test_word_ref_matches_token_case_insensitively(self): + assert matches_reference("12b", "exhibit-12B-scan.pdf") + assert not matches_reference("12b", "exhibit-12-scan.pdf") + + def test_delimiter_edges_yield_no_empty_token_match(self): + # A leading delimiter makes the split emit an empty first token, + # which must be skipped, not compared. + assert matches_reference("482", "_482_final.pdf") + assert not matches_reference("", "_482_final.pdf") + + def test_filename_ref_matches_whole_name(self): + assert matches_reference("survey_report.pdf", "survey_report.pdf") + assert matches_reference("survey_report.pdf", "archive/survey_report.pdf") + + +class TestParseAggregate: + def test_non_count_question_is_none(self): + assert parse_aggregate("what did the keeper record?") is None + + def test_term_mention_count(self): + agg = parse_aggregate("how many documents mention the observatory?") + assert agg is not None + assert agg.kind is AggregateKind.TERM_MENTIONS + assert agg.term == "observatory" + + def test_roughly_prefix(self): + agg = parse_aggregate("Roughly how many passages reference kerosene?") + assert agg is not None + assert agg.kind is AggregateKind.TERM_MENTIONS + assert agg.term == "kerosene" + + def test_total_sources(self): + agg = parse_aggregate("how many documents are there?") + assert agg is not None + assert agg.kind is AggregateKind.TOTAL_SOURCES + + def test_typed_count_is_unsupported_not_topical(self): + """Counts over records the store does not hold must be declined + precisely, not fed to retrieval that cannot count.""" + agg = parse_aggregate("how many shipments is each part number associated with?") + assert agg is not None + assert agg.kind is AggregateKind.UNSUPPORTED + + +class TestCountTermMentions: + @pytest.fixture() + def store(self, tmp_path): + from lilbee.core.config import cfg + from lilbee.data.store import Store + + config = cfg.model_copy(update={"lancedb_dir": tmp_path / "lancedb_test"}) + store = Store(config) + dim = config.embedding_dim + store.add_chunks( + [ + { + "source": f"doc{i}.md", + "content_type": "text", + "chunk_type": "raw", + "page_start": 0, + "page_end": 0, + "line_start": 0, + "line_end": 0, + "chunk": text, + "chunk_index": 0, + "vector": [0.1] * dim, + } + for i, text in enumerate( + [ + "the Lighthouse Trust funded the repair", + "no mention here", + "lighthouse trust grants were reviewed; the trust replied", + "unrelated text", + ] + ) + ] + ) + return store + + def test_counts_are_exact_and_case_insensitive(self, store): + chunk_hits, source_hits = store.count_term_mentions("lighthouse") + assert chunk_hits == 2 + assert source_hits == 2 + + def test_zero_for_absent_term(self, store): + assert store.count_term_mentions("zebra") == (0, 0) + + def test_empty_store_counts_zero(self, tmp_path): + from lilbee.core.config import cfg + from lilbee.data.store import Store + + config = cfg.model_copy(update={"lancedb_dir": tmp_path / "empty_lancedb"}) + empty = Store(config) + assert empty.count_term_mentions("anything") == (0, 0) + assert empty.count_chunks() == 0 + + def test_count_chunks(self, store): + assert store.count_chunks() == 4 diff --git a/tests/test_query.py b/tests/test_query.py index ddef0ba91..05840dc77 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1,5 +1,6 @@ """Tests for the RAG query pipeline (mocked: no live server needed).""" +from typing import ClassVar from unittest import mock import pytest @@ -82,6 +83,10 @@ def mock_svc(): set_services(None) +# Sentinel: derive the canonical score from distance unless overridden. +_AUTO_SCORE = object() + + def _make_result( source="test.pdf", content_type="pdf", @@ -97,7 +102,12 @@ def _make_result( bm25_score=None, rerank_score=None, vector=None, + score=_AUTO_SCORE, ) -> SearchChunk: + if score is _AUTO_SCORE: + # Mirror the store contract: every search path sets the canonical + # score. Pass score=None explicitly to build a legacy (pre-score) row. + score = max(0.0, min(1.0, 1.0 - distance)) if distance is not None else None return SearchChunk( source=source, content_type=content_type, @@ -113,6 +123,7 @@ def _make_result( bm25_score=bm25_score, rerank_score=rerank_score, vector=vector or [0.1], + score=score, ) @@ -240,16 +251,31 @@ def test_missing_distance_sorts_last(self): assert sorted_results[0].source == "has_dist.pdf" assert sorted_results[1].source == "no_dist.pdf" - def test_sorts_by_relevance_score_when_present(self): + def test_sorts_by_canonical_score_first(self): results = [ - _make_result(source="low.pdf", relevance_score=0.2), - _make_result(source="high.pdf", relevance_score=0.9), - _make_result(source="mid.pdf", relevance_score=0.5), + _make_result(source="low.pdf", score=0.2), + _make_result(source="high.pdf", score=0.9), + _make_result(source="mid.pdf", score=0.5), ] sorted_results = sort_by_relevance(results) - assert sorted_results[0].source == "high.pdf" - assert sorted_results[1].source == "mid.pdf" - assert sorted_results[2].source == "low.pdf" + assert [r.source for r in sorted_results] == ["high.pdf", "mid.pdf", "low.pdf"] + + def test_sorts_legacy_rows_by_relevance_score(self): + results = [ + _make_result(source="low.pdf", distance=None, score=None, relevance_score=0.2), + _make_result(source="high.pdf", distance=None, score=None, relevance_score=0.9), + _make_result(source="mid.pdf", distance=None, score=None, relevance_score=0.5), + ] + sorted_results = sort_by_relevance(results) + assert [r.source for r in sorted_results] == ["high.pdf", "mid.pdf", "low.pdf"] + + def test_sorts_legacy_rows_by_distance(self): + results = [ + _make_result(source="far.pdf", distance=0.9, score=None), + _make_result(source="near.pdf", distance=0.1, score=None), + ] + sorted_results = sort_by_relevance(results) + assert [r.source for r in sorted_results] == ["near.pdf", "far.pdf"] class TestDiversifySources: @@ -324,6 +350,59 @@ def test_numbers_chunks(self): assert "[2]" in ctx assert "chunk one" in ctx + def test_header_names_the_source_document(self): + """The answering model must see which document a block came from.""" + results = [ + _make_result( + source="survey_report.pdf", chunk="core sample B17", page_start=3, page_end=4 + ) + ] + ctx = build_context(results) + assert "[1] (survey_report.pdf, pages 3-4)" in ctx + assert "core sample B17" in ctx + + def test_header_shows_lines_for_code(self): + results = [ + _make_result( + source="app.py", content_type="code", chunk="def x():", line_start=10, line_end=20 + ) + ] + ctx = build_context(results) + assert "(app.py, lines 10-20)" in ctx + + def test_header_plain_for_text(self): + results = [_make_result(source="notes.md", content_type="text", chunk="hello")] + ctx = build_context(results) + assert "[1] (notes.md)\nhello" in ctx + + +class TestCitedIndexExtraction: + def test_single_brackets(self): + from lilbee.retrieval.query.formatting import _extract_cited_indices + + assert _extract_cited_indices("see [1] and [3]") == {1, 3} + + def test_comma_groups(self): + from lilbee.retrieval.query.formatting import _extract_cited_indices + + assert _extract_cited_indices("as shown [1, 2] and [4,5]") == {1, 2, 4, 5} + + def test_ranges(self): + from lilbee.retrieval.query.formatting import _extract_cited_indices + + assert _extract_cited_indices("documented across [1-3]") == {1, 2, 3} + + def test_mixed_group(self): + from lilbee.retrieval.query.formatting import _extract_cited_indices + + assert _extract_cited_indices("per [1, 3-5]") == {1, 3, 4, 5} + + def test_absurd_range_ignored(self): + from lilbee.retrieval.query.formatting import _extract_cited_indices + + # A page-span artifact like [1-500] must not fan out into 500 cites. + assert _extract_cited_indices("[1-500]") == set() + @pytest.mark.usefixtures("wiki_enabled") class TestSearchContext: @@ -392,6 +471,25 @@ def test_caps_at_three(self, mock_svc): ) assert len(variants) == 3 + def test_strips_reasoning_from_expansion_output(self, mock_svc): + """A reasoning chat model's think block must not become variants that + get embedded and searched.""" + mock_svc.provider.chat.return_value = _text_result( + "the user wants alternatives, let me consider\n" + "how do pods get scheduled\nkubernetes pod scheduling" + ) + variants = get_services().searcher._llm_expand("explain pod scheduling", 3) + assert variants == ["how do pods get scheduled", "kubernetes pod scheduling"] + + def test_strips_list_markers_from_variants(self, mock_svc): + """Models number their output despite the prompt; '1.' must not reach + the BM25 arm of the variant search.""" + mock_svc.provider.chat.return_value = _text_result( + "1. first phrasing\n2) second phrasing\n- third phrasing" + ) + variants = get_services().searcher._llm_expand("anything at all", 3) + assert variants == ["first phrasing", "second phrasing", "third phrasing"] + def test_returns_empty_on_error(self, mock_svc): mock_svc.provider.chat.side_effect = RuntimeError("no provider") assert ( @@ -1048,37 +1146,48 @@ def test_hyphenated_phrase_splits_into_tokens(self, mock_svc): class TestShouldSkipExpansion: - """bm25_probe rows carry a raw, unbounded BM25 score in ``bm25_score`` (LanceDB - FTS _score); the probe values here use those realistic magnitudes, not - pre-normalized [0, 1] numbers, since _should_skip_expansion sigmoid-squashes - them before comparing to the [0, 1] thresholds (default 0.8 / gap 0.15).""" + """bm25_probe rows carry a raw, unbounded BM25 score in ``bm25_score`` + (LanceDB FTS _score); the probe values here use those realistic + magnitudes. Confidence is the saturating s/(s+5) (0.8 = raw 20), and the + gap condition is the RELATIVE raw gap (top - second) / top >= 0.15: a + sigmoid squash compressed any two strong scores to within 0.01, so the + old gap test could never fire when the lexical arm was most certain.""" def test_skips_when_confident(self, mock_svc): - # sigmoid(5.0)=0.993 >= 0.8; gap to sigmoid(1.0)=0.731 is 0.262 >= 0.15. + # confidence(30) = 30/35 = 0.857 >= 0.8; relative gap (30-10)/30 = 0.67. mock_svc.store.bm25_probe.return_value = [ - _make_result(bm25_score=5.0), - _make_result(bm25_score=1.0), + _make_result(bm25_score=30.0), + _make_result(bm25_score=10.0), + ] + assert get_services().searcher._should_skip_expansion("test query") is True + + def test_skips_on_strong_scores_with_real_gap(self, mock_svc): + """The saturation regression case: two strong raw scores with a real + relative gap must skip; the sigmoid made this branch unreachable.""" + mock_svc.store.bm25_probe.return_value = [ + _make_result(bm25_score=40.0), + _make_result(bm25_score=25.0), ] assert get_services().searcher._should_skip_expansion("test query") is True def test_does_not_skip_when_low_score(self, mock_svc): - # sigmoid(0.5)=0.622 < 0.8: a weak top BM25 hit still expands. + # confidence(2) = 2/7 = 0.29 < 0.8: a weak top BM25 hit still expands. mock_svc.store.bm25_probe.return_value = [ - _make_result(bm25_score=0.5), - _make_result(bm25_score=0.4), + _make_result(bm25_score=2.0), + _make_result(bm25_score=1.0), ] assert get_services().searcher._should_skip_expansion("test query") is False def test_does_not_skip_when_close_gap(self, mock_svc): - # sigmoid(5.0)=0.993 and sigmoid(4.0)=0.982: gap 0.011 < 0.15. + # Strong but undifferentiated: relative gap (30-28)/30 = 0.07 < 0.15. mock_svc.store.bm25_probe.return_value = [ - _make_result(bm25_score=5.0), - _make_result(bm25_score=4.0), + _make_result(bm25_score=30.0), + _make_result(bm25_score=28.0), ] assert get_services().searcher._should_skip_expansion("test query") is False def test_skips_with_single_confident_result(self, mock_svc): - mock_svc.store.bm25_probe.return_value = [_make_result(bm25_score=5.0)] + mock_svc.store.bm25_probe.return_value = [_make_result(bm25_score=30.0)] assert get_services().searcher._should_skip_expansion("test") is True def test_does_not_skip_when_empty(self, mock_svc): @@ -1132,9 +1241,11 @@ def test_squashes_and_floors(self): assert _bm25_confidence(None) == 0.0 assert _bm25_confidence(0.0) == 0.0 assert _bm25_confidence(-2.0) == 0.0 - # Positive raw BM25 scores squash monotonically into (0.5, 1). - assert 0.5 < _bm25_confidence(1.0) < 1.0 + # Positive raw BM25 scores squash monotonically into (0, 1) without + # saturating: strong scores stay distinguishable from each other. + assert 0.0 < _bm25_confidence(1.0) < 1.0 assert _bm25_confidence(5.0) > _bm25_confidence(1.0) + assert _bm25_confidence(40.0) - _bm25_confidence(20.0) > 0.05 class TestParseStructuredQuery: @@ -1193,6 +1304,20 @@ def test_returns_empty_on_blank(self, mock_svc): mock_svc.provider.chat.return_value = _text_result(" ") assert get_services().searcher._hyde_search("test", top_k=5) == [] + def test_strips_reasoning_before_embedding(self, mock_svc): + """A reasoning model's deliberation must not be embedded as the + hypothetical passage.""" + mock_svc.provider.chat.return_value = _text_result( + "what would a real document say herethe actual passage" + ) + mock_svc.store.search.return_value = [] + get_services().searcher._hyde_search("explain X", top_k=5) + mock_svc.embedder.embed_query.assert_called_once_with("the actual passage") + + def test_returns_empty_when_output_is_all_reasoning(self, mock_svc): + mock_svc.provider.chat.return_value = _text_result("only deliberation") + assert get_services().searcher._hyde_search("test", top_k=5) == [] + class TestTemporalFilter: def test_filters_by_date(self, mock_svc): @@ -1336,11 +1461,11 @@ def test_structured_term_mode(self, mock_svc): def test_skips_expansion_when_confident(self, mock_svc): mock_svc.store.search.return_value = [_make_result()] - # Raw BM25 magnitudes (sigmoid(5.0)=0.993, sigmoid(1.0)=0.731): confident - # top with a wide gap, so expansion is skipped. + # Raw BM25 magnitudes: confidence(30) = 0.857 >= 0.8 with a wide + # relative gap ((30-10)/30 = 0.67), so expansion is skipped. mock_svc.store.bm25_probe.return_value = [ - _make_result(bm25_score=5.0), - _make_result(bm25_score=1.0), + _make_result(bm25_score=30.0), + _make_result(bm25_score=10.0), ] results = get_services().searcher.search("exact match query") # Provider.chat should NOT be called for expansion @@ -1380,8 +1505,9 @@ def fake_hyde(question, top_k, chunk_type=None): finally: cfg.hyde, cfg.concept_graph, cfg.query_expansion_count = old_hyde, old_cg, old_qe - def test_hyde_adds_unique_results_with_distance_adjustment(self, mock_svc): - """HyDE results not seen in normal search are added with adjusted distance.""" + def test_hyde_adds_unique_results_downweighted_in_score_space(self, mock_svc): + """HyDE results not seen in normal search are added with their canonical + score discounted by hyde_weight; distance provenance stays untouched.""" normal_result = _make_result(source="normal.md", chunk_index=0) hyde_only_result = _make_result(source="hyde.md", chunk_index=0, distance=0.8) mock_svc.store.search.side_effect = [ @@ -1400,13 +1526,245 @@ def test_hyde_adds_unique_results_with_distance_adjustment(self, mock_svc): assert "normal.md" in sources assert "hyde.md" in sources hyde_r = next(r for r in results if r.source == "hyde.md") - assert hyde_r.distance == pytest.approx(0.8 / 0.5) + assert hyde_r.score == pytest.approx((1.0 - 0.8) * 0.5) + assert hyde_r.distance == pytest.approx(0.8) finally: cfg.query_expansion_count = 3 cfg.hyde = False cfg.hyde_weight = 0.7 +class TestKnownItemRoute: + def _source(self, filename): + return {"filename": filename, "file_hash": "h", "ingested_at": "", "chunk_count": 2} + + def test_named_document_bypasses_similarity_search(self, mock_svc): + """A question naming one resolvable document gets that document's + chunks in document order, not a similarity ranking.""" + mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] + mock_svc.store.get_chunks_by_source.return_value = [ + _make_result(source="survey_report.pdf", chunk="second", chunk_index=1), + _make_result(source="survey_report.pdf", chunk="first", chunk_index=0), + ] + rag = get_services().searcher.build_rag_context("summarize survey_report.pdf") + assert rag is not None + results, _ = rag + assert [r.chunk for r in results] == ["first", "second"] + assert all(r.score == 1.0 for r in results) + mock_svc.store.search.assert_not_called() + + def test_numeric_reference_resolves_against_zero_padded_ids(self, mock_svc): + """The private-corpus failure shape: a bare number must resolve to the + one zero-padded id that token-matches it, not fall back to topical + because substring search also hit a longer number.""" + mock_svc.store.get_sources.return_value = [ + self._source("ARC-REC-00000482.pdf"), + self._source("ARC-REC-00010482.pdf"), + ] + mock_svc.store.get_chunks_by_source.return_value = [ + _make_result(source="ARC-REC-00000482.pdf", chunk="body", chunk_index=0) + ] + rag = get_services().searcher.build_rag_context("summarize document 482") + assert rag is not None + mock_svc.store.get_chunks_by_source.assert_called_once_with("ARC-REC-00000482.pdf") + mock_svc.store.search.assert_not_called() + + def test_quoted_title_resolves_via_unique_substring_match(self, mock_svc): + """A quoted multi-word title never token-matches a hyphenated + filename; when substring search finds exactly one source, that + uniqueness still routes.""" + mock_svc.store.get_sources.return_value = [self._source("harbor-survey-2010.pdf")] + mock_svc.store.get_chunks_by_source.return_value = [ + _make_result(source="harbor-survey-2010.pdf", chunk="body", chunk_index=0) + ] + rag = get_services().searcher.build_rag_context('summarize "harbor survey 2010"') + assert rag is not None + mock_svc.store.get_chunks_by_source.assert_called_once_with("harbor-survey-2010.pdf") + + def test_numeric_ref_with_only_substring_hit_stays_topical(self, mock_svc): + """ "12" inside "notes-2012" is a false match; a numeric reference that + token-matches nothing must not route via the substring fallback.""" + mock_svc.store.get_sources.return_value = [self._source("notes-2012.pdf")] + mock_svc.store.search.return_value = [_make_result()] + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context("summarize document 12") + finally: + cfg.query_expansion_count = 3 + mock_svc.store.get_chunks_by_source.assert_not_called() + mock_svc.store.search.assert_called_once() + + def test_ambiguous_reference_falls_back_to_topical(self, mock_svc): + mock_svc.store.get_sources.return_value = [ + self._source("report_12a.pdf"), + self._source("report_12b.pdf"), + ] + mock_svc.store.search.return_value = [_make_result()] + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context("summarize document 12") + finally: + cfg.query_expansion_count = 3 + mock_svc.store.search.assert_called_once() + + def test_disabled_by_config(self, mock_svc): + mock_svc.store.search.return_value = [_make_result()] + cfg.intent_routing = False + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context("summarize survey_report.pdf") + finally: + cfg.intent_routing = True + cfg.query_expansion_count = 3 + mock_svc.store.get_sources.assert_not_called() + mock_svc.store.search.assert_called_once() + + def test_resolved_source_with_no_chunks_falls_back(self, mock_svc): + mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] + mock_svc.store.get_chunks_by_source.return_value = [] + mock_svc.store.search.return_value = [_make_result()] + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context("summarize survey_report.pdf") + finally: + cfg.query_expansion_count = 3 + mock_svc.store.search.assert_called_once() + + +class TestAggregateRoute: + def test_term_count_answers_without_llm(self, mock_svc): + """A count question gets an exact scan answer; no model is called, so + no model can hedge or invent the number.""" + mock_svc.store.count_term_mentions.return_value = (412, 57) + result = get_services().searcher.ask_raw("how many documents mention the observatory?") + assert "57 documents" in result.answer + assert "412 passages" in result.answer + mock_svc.provider.chat.assert_not_called() + mock_svc.store.search.assert_not_called() + + def test_total_count(self, mock_svc): + mock_svc.store.count_sources.return_value = 369 + mock_svc.store.count_chunks.return_value = 123456 + result = get_services().searcher.ask_raw("how many documents are there?") + assert "369" in result.answer + assert "123456" in result.answer + + def test_typed_count_declines_precisely(self, mock_svc): + result = get_services().searcher.ask_raw( + "how many shipments is each part number associated with?" + ) + assert "aren't extracted" in result.answer + mock_svc.provider.chat.assert_not_called() + + def test_stream_path_routes_aggregates_too(self, mock_svc): + mock_svc.store.count_term_mentions.return_value = (10, 3) + tokens = list(get_services().searcher.ask_stream("how many documents mention kerosene?")) + text = "".join(t.content for t in tokens) + assert "3 documents" in text + mock_svc.provider.chat.assert_not_called() + + def test_topical_question_unaffected(self, mock_svc): + mock_svc.store.search.return_value = [_make_result()] + mock_svc.provider.chat.return_value = _text_result("an answer [1]") + cfg.query_expansion_count = 0 + try: + result = get_services().searcher.ask_raw("what did the keeper record in October?") + finally: + cfg.query_expansion_count = 3 + assert result.sources + mock_svc.store.count_term_mentions.assert_not_called() + + def test_disabled_by_config_goes_topical(self, mock_svc): + mock_svc.store.search.return_value = [_make_result()] + mock_svc.provider.chat.return_value = _text_result("an answer [1]") + cfg.intent_routing = False + cfg.query_expansion_count = 0 + try: + get_services().searcher.ask_raw("how many documents mention the observatory?") + finally: + cfg.intent_routing = True + cfg.query_expansion_count = 3 + mock_svc.store.count_term_mentions.assert_not_called() + mock_svc.store.search.assert_called_once() + + +class TestHistoryCondensation: + _HISTORY: ClassVar[list[dict[str, str]]] = [ + {"role": "user", "content": "who kept the lighthouse journal at Split Rock"}, + {"role": "assistant", "content": "It was kept by E. Larsen [1]."}, + ] + + def test_follow_up_is_rewritten_for_retrieval(self, mock_svc): + """Retrieval must see the standalone rewrite, not the pronouns; the + answering prompt keeps the user's original wording.""" + rewritten = "when was the Split Rock lighthouse journal written" + mock_svc.provider.chat.return_value = _text_result(rewritten) + mock_svc.store.search.return_value = [_make_result()] + cfg.query_expansion_count = 0 + try: + rag = get_services().searcher.build_rag_context( + "and when was it written?", history=list(self._HISTORY) + ) + finally: + cfg.query_expansion_count = 3 + assert rag is not None + _, messages = rag + assert mock_svc.store.search.call_args[1]["query_text"] == rewritten + assert "and when was it written?" in messages[-1]["content"] + assert rewritten not in messages[-1]["content"] + + def test_no_history_means_no_rewrite_call(self, mock_svc): + mock_svc.store.search.return_value = [_make_result()] + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context("standalone question") + finally: + cfg.query_expansion_count = 3 + mock_svc.provider.chat.assert_not_called() + + def test_falls_back_to_original_on_provider_error(self, mock_svc): + mock_svc.provider.chat.side_effect = RuntimeError("no provider") + mock_svc.store.search.return_value = [_make_result()] + cfg.query_expansion_count = 0 + try: + rag = get_services().searcher.build_rag_context( + "and when was it written?", history=list(self._HISTORY) + ) + finally: + cfg.query_expansion_count = 3 + assert rag is not None + assert mock_svc.store.search.call_args[1]["query_text"] == "and when was it written?" + + def test_disabled_by_config(self, mock_svc): + cfg.history_rewrite = False + cfg.query_expansion_count = 0 + mock_svc.store.search.return_value = [_make_result()] + try: + get_services().searcher.build_rag_context( + "and when was it written?", history=list(self._HISTORY) + ) + finally: + cfg.history_rewrite = True + cfg.query_expansion_count = 3 + mock_svc.provider.chat.assert_not_called() + assert mock_svc.store.search.call_args[1]["query_text"] == "and when was it written?" + + def test_reasoning_stripped_from_rewrite(self, mock_svc): + mock_svc.provider.chat.return_value = _text_result( + "the user means the journalwhen was the Split Rock journal written" + ) + mock_svc.store.search.return_value = [_make_result()] + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context( + "and when was it written?", history=list(self._HISTORY) + ) + finally: + cfg.query_expansion_count = 3 + expected = "when was the Split Rock journal written" + assert mock_svc.store.search.call_args[1]["query_text"] == expected + + class TestAskRawWithReranker: def test_reranker_called_when_configured(self, mock_svc): mock_svc.store.search.return_value = [_make_result()] @@ -2016,6 +2374,14 @@ def test_drops_high_distance(self): assert len(filtered) == 1 assert filtered[0].source == "close.pdf" + def test_drops_high_distance_legacy_rows(self): + results = [ + _make_result(source="close.pdf", distance=0.3, score=None), + _make_result(source="far.pdf", distance=0.95, score=None, chunk_index=1), + ] + filtered = filter_results(results, max_distance=0.9) + assert [r.source for r in filtered] == ["close.pdf"] + def test_drops_low_relevance_score(self): results = [ _make_result(source="good.pdf", distance=None, relevance_score=0.8), @@ -2035,6 +2401,26 @@ def test_disabled_when_zero(self): filtered = filter_results(results, max_distance=0, min_relevance_score=0) assert len(filtered) == 1 + def test_canonical_score_gates_on_min_relevance(self): + """min_relevance_score is a real abstention threshold against the + canonical [0,1] score; with every row below it, retrieval comes back + empty and ask() can refuse instead of feeding noise as context.""" + results = [ + _make_result(source="noise1.pdf", score=0.04), + _make_result(source="noise2.pdf", score=0.02, chunk_index=1), + ] + assert filter_results(results, max_distance=0.9, min_relevance_score=0.1) == [] + + def test_lexically_supported_row_survives_far_distance(self): + """A row the BM25 arm matched keeps its standing regardless of vector + distance; only unsupported far rows are dropped.""" + results = [ + _make_result(source="identifier.pdf", distance=1.4, score=0.35, bm25_score=30.0), + _make_result(source="drift.pdf", distance=1.4, score=0.1, chunk_index=1), + ] + filtered = filter_results(results, max_distance=0.9) + assert [r.source for r in filtered] == ["identifier.pdf"] + def test_keeps_results_at_threshold(self): r = _make_result(distance=0.9) filtered = filter_results([r], max_distance=0.9) @@ -2054,10 +2440,18 @@ def test_neither_returns_default(self): r = _make_result(distance=None, relevance_score=None) assert _relevance_weight(r) == pytest.approx(0.5) - def test_relevance_score_takes_priority(self): - r = _make_result(distance=0.3, relevance_score=0.9) + def test_canonical_score_takes_priority(self): + r = _make_result(distance=0.3, relevance_score=0.9, score=0.6) + assert _relevance_weight(r) == pytest.approx(0.6) + + def test_legacy_relevance_score_beats_distance(self): + r = _make_result(distance=0.3, relevance_score=0.9, score=None) assert _relevance_weight(r) == pytest.approx(0.9) + def test_legacy_distance_inverts_when_only_signal(self): + r = _make_result(distance=0.3, relevance_score=None, score=None) + assert _relevance_weight(r) == pytest.approx(0.7) + def test_clamps_high_relevance(self): r = _make_result(distance=None, relevance_score=1.5) assert _relevance_weight(r) == pytest.approx(1.0) @@ -2067,6 +2461,45 @@ def test_clamps_negative_distance(self): assert _relevance_weight(r) == pytest.approx(0.0) +class TestCitedSubsetByName: + def test_name_mention_counts_as_citation(self): + from lilbee.retrieval.query.formatting import cited_subset + + sources = [ + _make_result(source="ARC-REC-00000482.pdf", chunk_index=0), + _make_result(source="survey_report.pdf", chunk_index=1), + ] + answer = "According to ARC-REC-00000482.pdf, the request was approved." + assert [c.source for c in cited_subset(answer, sources)] == ["ARC-REC-00000482.pdf"] + + def test_stem_mention_counts_when_identifier_shaped(self): + from lilbee.retrieval.query.formatting import cited_subset + + sources = [_make_result(source="ARC-REC-00000482.pdf", chunk_index=0)] + answer = "The memo in ARC-REC-00000482 approves the request." + assert len(cited_subset(answer, sources)) == 1 + + def test_prose_word_stem_does_not_false_positive(self): + from lilbee.retrieval.query.formatting import cited_subset + + sources = [_make_result(source="notes.md", chunk_index=0)] + answer = "The witness notes that the repair happened in March." + assert cited_subset(answer, sources) == [] + + def test_marker_and_name_citations_combine(self): + from lilbee.retrieval.query.formatting import cited_subset + + sources = [ + _make_result(source="alpha_report.pdf", chunk_index=0), + _make_result(source="beta_summary.pdf", chunk_index=1), + ] + answer = "Per [2], and as alpha_report.pdf notes, both agree." + assert {c.source for c in cited_subset(answer, sources)} == { + "alpha_report.pdf", + "beta_summary.pdf", + } + + class TestStripLlmCitations: def test_removes_sources_block(self): text = "The answer is 42.\n\nSources:\n- test.pdf, page 5" diff --git a/tests/test_server_handlers.py b/tests/test_server_handlers.py index a59359d16..5b1274264 100644 --- a/tests/test_server_handlers.py +++ b/tests/test_server_handlers.py @@ -73,6 +73,8 @@ def mock_svc(): # Default to the grounded retrieval path; mode/embedder tests flip these. searcher.skip_retrieval.return_value = False searcher.search_unavailable.return_value = False + # No direct (count-scan) answer unless a test routes one explicitly. + searcher.route_direct_answer.return_value = None services = make_mock_services(searcher=searcher) # chat_dispatch validates cfg.chat_model against the registry. chat_manifest = MagicMock() @@ -371,6 +373,17 @@ async def test_no_results_yields_error(self, mock_svc): parsed = json.loads(non_empty[0].split("data: ")[1].strip()) assert "No relevant documents found" in parsed["message"] + async def test_direct_answer_streams_before_retrieval(self, mock_svc): + """Count questions stream the exact-scan answer, mirroring + Searcher.ask_stream, instead of hedging through top-k RAG.""" + mock_svc.searcher.route_direct_answer.return_value = "Exact scan: 3 documents." + events = [e async for e in handlers.ask_stream("how many documents mention x?")] + non_empty = [e for e in events if e] + token_event = next(e for e in non_empty if e.startswith("event: token")) + assert "Exact scan: 3 documents." in token_event + assert any(e.startswith("event: done") for e in non_empty) + mock_svc.searcher.build_rag_context.assert_not_called() + async def test_yields_token_sources_done(self, mock_svc): mock_svc.searcher.build_rag_context.return_value = _rag_return() mock_svc.provider.chat.return_value = iter(["answer"]) @@ -612,6 +625,15 @@ async def _gen(): class TestChat: + async def test_direct_answer_routes_before_retrieval(self, mock_svc): + """A count question answered by the exact scan must short-circuit the + HTTP chat path exactly as it does ask_raw: same router, no LLM.""" + mock_svc.searcher.route_direct_answer.return_value = "Exact scan: 3 documents." + result = await handlers.chat("how many documents mention kerosene?", []) + assert result.answer == "Exact scan: 3 documents." + assert result.sources == [] + mock_svc.searcher.build_rag_context.assert_not_called() + async def test_passes_history(self, mock_svc, monkeypatch): from lilbee.server.chat_dispatch.canonical import ( CanonicalResponse, @@ -826,6 +848,14 @@ async def test_no_results_yields_error(self, mock_svc): non_empty = [e for e in events if e] assert any("error" in e for e in non_empty) + async def test_direct_answer_streams_before_retrieval(self, mock_svc): + mock_svc.searcher.route_direct_answer.return_value = "Exact scan: 3 documents." + events = [e async for e in handlers.chat_stream("how many documents mention x?", [])] + non_empty = [e for e in events if e] + token_event = next(e for e in non_empty if e.startswith("event: token")) + assert "Exact scan: 3 documents." in token_event + mock_svc.searcher.build_rag_context.assert_not_called() + async def test_yields_events_with_history(self, mock_svc, monkeypatch): mock_svc.searcher.build_rag_context.return_value = _rag_return() monkeypatch.setattr( diff --git a/tests/test_server_litestar.py b/tests/test_server_litestar.py index c257888a4..c070fbf7a 100644 --- a/tests/test_server_litestar.py +++ b/tests/test_server_litestar.py @@ -657,6 +657,8 @@ async def _collect(): # embedder present (search available, retrieval runs). searcher.search_unavailable.return_value = False searcher.skip_retrieval.return_value = False + # Not a count question: the direct-answer route must pass. + searcher.route_direct_answer.return_value = None searcher.build_rag_context.side_effect = self._mismatch() return [event async for event in make_stream(rag)] diff --git a/tests/test_store.py b/tests/test_store.py index 2c1123233..ce4e0b67a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -598,7 +598,9 @@ def test_hybrid_search_with_fts_index(self, store, test_config): query_vec = [0.5] * test_config.embedding_dim results = store.search(query_vec, top_k=3, query_text="chunk number") assert len(results) > 0 - assert results[0].relevance_score is not None + assert all(r.score is not None for r in results) + scores = [r.score for r in results] + assert all(0.0 <= s <= 1.0 for s in scores) def test_fallback_to_vector_when_no_query_text(self, store, test_config): records = _make_records() @@ -618,15 +620,20 @@ def test_fallback_to_vector_when_no_fts_index(self, store, test_config): assert len(results) > 0 assert results[0].distance is not None - def test_hybrid_fallback_on_exception(self, store, test_config): + def test_hybrid_fallback_on_exception(self, store, test_config, caplog): records = _make_records() store.add_chunks(records) store.ensure_fts_index() query_vec = [0.5] * test_config.embedding_dim - with mock.patch("lilbee.data.store.core._hybrid_search", side_effect=RuntimeError("boom")): + with ( + mock.patch.object(store, "_hybrid_search", side_effect=RuntimeError("boom")), + caplog.at_level("WARNING", logger="lilbee.data.store.core"), + ): results = store.search(query_vec, top_k=3, query_text="chunk") assert len(results) > 0 assert results[0].distance is not None + # The downgrade changes recall for the query; it must be visible. + assert any("falling back to vector-only" in r.message for r in caplog.records) def test_vector_only_applies_mmr(self, store, test_config): """Vector-only path (no query_text) applies MMR when results > top_k.""" @@ -636,6 +643,72 @@ def test_vector_only_applies_mmr(self, store, test_config): results = store.search(query_vec, top_k=2) assert len(results) == 2 + def test_vector_only_results_carry_canonical_score(self, store, test_config): + records = _make_records() + store.add_chunks(records) + query_vec = [0.5] * test_config.embedding_dim + results = store.search(query_vec, top_k=3) + assert all(r.score is not None for r in results) + # Higher similarity (lower distance) must mean higher canonical score. + pairs = [(r.distance, r.score) for r in results] + assert sorted(pairs, key=lambda p: p[0]) == sorted(pairs, key=lambda p: p[1], reverse=True) + + def test_lexical_only_match_survives_hybrid(self, store, test_config): + """A chunk only BM25 can find (its vector points away from the query) + must appear in hybrid results: the identifier-query case.""" + records = _make_records(n=30) + outlier_vec = [-1.0] * test_config.embedding_dim + records.append( + { + "source": "parts_catalog_214.pdf", + "content_type": "pdf", + "chunk_type": "raw", + "page_start": 1, + "page_end": 1, + "line_start": 0, + "line_end": 0, + "chunk": "part number PX4471 shipped from fresno", + "chunk_index": 0, + "vector": outlier_vec, + } + ) + store.add_chunks(records) + store.ensure_fts_index() + query_vec = [0.5] * test_config.embedding_dim + results = store.search(query_vec, top_k=5, query_text="part number PX4471") + assert any(r.source == "parts_catalog_214.pdf" for r in results) + + def test_hybrid_overfetch_floor(self, store, test_config): + """Each arm is fetched at least fusion_overfetch_floor deep.""" + records = _make_records(n=3) + store.add_chunks(records) + store.ensure_fts_index() + query_vec = [0.5] * test_config.embedding_dim + with ( + mock.patch.object(store, "_vector_arm", return_value=[]) as vec_arm, + mock.patch.object(store, "_fts_arm", return_value=[]) as fts_arm, + ): + store.search(query_vec, top_k=3, query_text="chunk") + assert vec_arm.call_args[0][2] == test_config.fusion_overfetch_floor + assert fts_arm.call_args[0][2] == test_config.fusion_overfetch_floor + + def test_hybrid_vector_arm_applies_ann_recovery(self, store, test_config): + """With a vector index present, the hybrid vector arm probes extra + partitions and refines against full vectors.""" + chain = mock.MagicMock() + chain.to_list.return_value = [] + table = mock.MagicMock() + table.search.return_value = chain + table.count_rows.return_value = 1_000_000 + chain.metric.return_value = chain + chain.limit.return_value = chain + chain.nprobes.return_value = chain + chain.refine_factor.return_value = chain + with mock.patch("lilbee.data.store.core._has_vector_index", return_value=True): + store._vector_arm(table, [0.5] * test_config.embedding_dim, 50, None) + chain.nprobes.assert_called_once() + chain.refine_factor.assert_called_once() + def test_auto_ensures_fts_index_when_query_text(self, store, test_config): records = _make_records() store.add_chunks(records) diff --git a/tests/test_store_fusion.py b/tests/test_store_fusion.py new file mode 100644 index 000000000..f9993abbd --- /dev/null +++ b/tests/test_store_fusion.py @@ -0,0 +1,121 @@ +"""Tests for score-aware fusion of the vector and BM25 arms.""" + +import pytest + +from lilbee.data.store import SearchChunk +from lilbee.data.store.fusion import fuse_arms, normalized_bm25, vector_similarity + + +def _chunk(source, idx, *, distance=None, bm25=None, dim=4): + return SearchChunk( + source=source, + content_type="text", + chunk_type="raw", + page_start=0, + page_end=0, + line_start=0, + line_end=0, + chunk=f"{source}:{idx}", + chunk_index=idx, + vector=[0.1] * dim, + distance=distance, + bm25_score=bm25, + ) + + +class TestVectorSimilarity: + def test_zero_distance_is_perfect(self): + assert vector_similarity(0.0) == 1.0 + + def test_opposite_vectors_clamp_to_zero(self): + assert vector_similarity(2.0) == 0.0 + + def test_midrange(self): + assert vector_similarity(0.4) == pytest.approx(0.6) + + +class TestNormalizedBm25: + def test_top_score_anchors_to_one(self): + assert normalized_bm25([20.0, 10.0, 5.0]) == pytest.approx([1.0, 0.5, 0.25]) + + def test_empty_list(self): + assert normalized_bm25([]) == [] + + def test_non_positive_scores_map_to_zero(self): + assert normalized_bm25([0.0, -1.0]) == [0.0, 0.0] + + +class TestFuseArms: + def test_both_arms_beat_either_alone(self): + both = _chunk("a.md", 0, distance=0.3) + vec_only = _chunk("b.md", 0, distance=0.3) + lex = _chunk("a.md", 0, bm25=12.0) + fused = fuse_arms([both, vec_only], [lex], alpha=0.6) + scores = {(r.source, r.chunk_index): r.score for r in fused} + assert scores[("a.md", 0)] > scores[("b.md", 0)] + + def test_lexical_only_row_survives_with_score(self): + """The identifier case: a row invisible to the vector arm must carry + real fused weight from its BM25 strength alone.""" + vec = [_chunk("noise.md", i, distance=0.5) for i in range(3)] + lex = [_chunk("catalog_482.pdf", 0, bm25=35.0)] + fused = fuse_arms(vec, lex, alpha=0.6) + lexical_row = next(r for r in fused if r.source == "catalog_482.pdf") + assert lexical_row.score == pytest.approx(0.4) + + def test_certain_lexical_hit_outranks_mediocre_dense_neighbors(self): + """The pinpoint-document failure mode, inverted: top BM25 with weak + vector support must beat vector rows at middling similarity.""" + vec = [_chunk("noise.md", i, distance=0.8) for i in range(5)] + lex = [_chunk("target.pdf", 0, bm25=30.0), _chunk("noise.md", 0, bm25=3.0)] + fused = fuse_arms(vec, lex, alpha=0.5) + assert fused[0].source == "target.pdf" + + def test_alpha_one_is_pure_vector(self): + fused = fuse_arms( + [_chunk("v.md", 0, distance=0.2)], [_chunk("l.md", 0, bm25=50.0)], alpha=1.0 + ) + scores = {r.source: r.score for r in fused} + assert scores["v.md"] == pytest.approx(0.8) + assert scores["l.md"] == 0.0 + + def test_alpha_zero_is_pure_lexical(self): + fused = fuse_arms( + [_chunk("v.md", 0, distance=0.2)], [_chunk("l.md", 0, bm25=50.0)], alpha=0.0 + ) + scores = {r.source: r.score for r in fused} + assert scores["l.md"] == pytest.approx(1.0) + assert scores["v.md"] == 0.0 + + def test_dedup_keeps_both_provenance_fields(self): + fused = fuse_arms( + [_chunk("a.md", 0, distance=0.3)], [_chunk("a.md", 0, bm25=9.0)], alpha=0.5 + ) + assert len(fused) == 1 + assert fused[0].distance == pytest.approx(0.3) + assert fused[0].bm25_score == pytest.approx(9.0) + + def test_sorted_descending_by_score(self): + fused = fuse_arms( + [_chunk("far.md", 0, distance=1.5), _chunk("near.md", 0, distance=0.1)], + [], + alpha=1.0, + ) + assert [r.source for r in fused] == ["near.md", "far.md"] + assert all(r.score is not None for r in fused) + + +class TestDropUnsupportedFarRows: + def test_disabled_when_max_distance_zero(self): + from lilbee.data.store.core import _drop_unsupported_far_rows + + rows = [_chunk("far.md", 0, distance=1.9)] + assert _drop_unsupported_far_rows(rows, 0.0) == rows + + def test_drops_vector_only_far_row_keeps_supported(self): + from lilbee.data.store.core import _drop_unsupported_far_rows + + far_unsupported = _chunk("drift.md", 0, distance=1.4) + far_supported = _chunk("identifier.md", 0, distance=1.4, bm25=25.0) + kept = _drop_unsupported_far_rows([far_unsupported, far_supported], 0.75) + assert [r.source for r in kept] == ["identifier.md"] From bb2347bd2a9621549dd5dbe78f599a6a49ca95d9 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:08:28 -0400 Subject: [PATCH 52/77] retrieval: resolve document references by content when filenames don't know them The evaluation handoff identified the remaining pinpoint gap: a docket-style reference lives in a document's text, and often nowhere in its filename, so filename-token resolution can never route it. When no filename resolves, a BM25 probe for the reference now routes to the single source that owns at least three quarters of the top hits; a reference scattered across documents (a number cited in related filings) stays topical, and several filenames legitimately carrying the token (a split document) remain ambiguous without probing. --- src/lilbee/retrieval/query/searcher.py | 63 ++++++++++++++++++++------ tests/test_query.py | 50 ++++++++++++++++++++ 2 files changed, 98 insertions(+), 15 deletions(-) diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index 88bc6e6a1..3d8674900 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -77,6 +77,14 @@ # disambiguation picks the unique winner. _KNOWN_ITEM_CANDIDATES = 50 +# Content-based known-item resolution: BM25 hits probed for a reference, and +# the fraction of them one source must own to count as that document. A +# docket-style number lives in a document's text, not its filename, so +# filename matching alone can never resolve it; concentration keeps the +# fallback conservative, since a number cited across many filings spreads. +_KNOWN_ITEM_PROBE_K = 6 +_KNOWN_ITEM_PROBE_MAJORITY = 0.75 + # Structured-query mode names (the ``mode:`` prefix shortcut). Single source for # both the prefix parser and the dispatch in ``_search_structured``. "term"/"vec"/ # "hyde" pick a retrieval strategy; "wiki"/"raw" are ChunkType scope shortcuts. @@ -571,21 +579,9 @@ def _known_item_results(self, question: str) -> list[SearchChunk]: relevance is established by the name match, not by similarity. """ for ref in document_references(question): - candidates = self._store.get_sources(search=ref, limit=_KNOWN_ITEM_CANDIDATES) - # Substring search over-matches (a bare "482" hits every - # zero-padded id containing it); token-exact matching does the - # disambiguation, and only a unique winner routes. - matches = [s for s in candidates if matches_reference(ref, s["filename"])] - if len(matches) != 1: - # A unique substring hit still routes for word refs (quoted - # titles never token-match hyphenated filenames), but not for - # numeric ones: "12" inside "notes-2012" is the exact false - # match token comparison exists to reject. - if len(candidates) == 1 and not ref.strip().isdigit(): - matches = candidates - else: - continue - filename = matches[0]["filename"] + filename = self._resolve_reference_filename(ref) + if filename is None: + continue chunks = self._store.get_chunks_by_source(filename) if not chunks: continue @@ -594,6 +590,43 @@ def _known_item_results(self, question: str) -> list[SearchChunk]: return [c.model_copy(update={"score": 1.0}) for c in chunks] return [] + def _resolve_reference_filename(self, ref: str) -> str | None: + """The one source *ref* names, or ``None`` when nothing resolves uniquely. + + Filename resolution first: substring search over-matches (a bare + "482" hits every zero-padded id containing it), so token-exact + matching disambiguates and only a unique winner routes. A unique + substring hit still routes for word refs (quoted titles never + token-match hyphenated filenames) but not numeric ones: "12" inside + "notes-2012" is the false match token comparison exists to reject. + + When no filename knows the reference, it may be a docket-style number + living in the document's own text; a BM25 probe resolves it when the + hits concentrate in a single source. + """ + candidates = self._store.get_sources(search=ref, limit=_KNOWN_ITEM_CANDIDATES) + matches = [s for s in candidates if matches_reference(ref, s["filename"])] + if len(matches) == 1: + return str(matches[0]["filename"]) + if not matches and len(candidates) == 1 and not ref.strip().isdigit(): + return str(candidates[0]["filename"]) + if matches: + return None # several sources genuinely carry the reference + return self._resolve_reference_by_content(ref) + + def _resolve_reference_by_content(self, ref: str) -> str | None: + """Resolve *ref* to the single source whose text owns it, if any.""" + hits = self._store.bm25_probe(ref, top_k=_KNOWN_ITEM_PROBE_K) + if len(hits) < _KNOWN_ITEM_PROBE_K: + return None + counts: dict[str, int] = {} + for hit in hits: + counts[hit.source] = counts.get(hit.source, 0) + 1 + top_source, owned = max(counts.items(), key=lambda kv: kv[1]) + if owned / len(hits) >= _KNOWN_ITEM_PROBE_MAJORITY: + return top_source + return None + def build_rag_context( self, question: str, diff --git a/tests/test_query.py b/tests/test_query.py index 05840dc77..6638c29ec 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1581,6 +1581,56 @@ def test_quoted_title_resolves_via_unique_substring_match(self, mock_svc): assert rag is not None mock_svc.store.get_chunks_by_source.assert_called_once_with("harbor-survey-2010.pdf") + def test_docket_reference_resolves_by_content_concentration(self, mock_svc): + """A reference that appears in a document's text but not its filename + (a docket number) resolves when BM25 hits concentrate in one source.""" + mock_svc.store.get_sources.return_value = [] + mock_svc.store.bm25_probe.return_value = [ + _make_result(source="scan_aa17.pdf", chunk_index=i, bm25_score=20.0 - i) + for i in range(6) + ] + mock_svc.store.get_chunks_by_source.return_value = [ + _make_result(source="scan_aa17.pdf", chunk="body", chunk_index=0) + ] + rag = get_services().searcher.build_rag_context("summarize document 482") + assert rag is not None + mock_svc.store.get_chunks_by_source.assert_called_once_with("scan_aa17.pdf") + mock_svc.store.search.assert_not_called() + + def test_multiple_token_owners_stay_topical_without_probing(self, mock_svc): + """Two files legitimately carrying the same reference token (a split + document) are true ambiguity: no route, and no content probe either, + since the filenames already prove the reference is shared.""" + mock_svc.store.get_sources.return_value = [ + self._source("ARC-482-part1.pdf"), + self._source("ARC-482-part2.pdf"), + ] + mock_svc.store.search.return_value = [_make_result()] + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context("summarize document 482") + finally: + cfg.query_expansion_count = 3 + mock_svc.store.get_chunks_by_source.assert_not_called() + mock_svc.store.search.assert_called_once() + + def test_scattered_content_hits_stay_topical(self, mock_svc): + """A reference mentioned across many documents (a number cited in + related filings) must not route: concentration is the signal.""" + mock_svc.store.get_sources.return_value = [] + mock_svc.store.bm25_probe.return_value = [ + _make_result(source=f"scan_{i}.pdf", chunk_index=0, bm25_score=20.0 - i) + for i in range(6) + ] + mock_svc.store.search.return_value = [_make_result()] + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context("summarize document 482") + finally: + cfg.query_expansion_count = 3 + mock_svc.store.get_chunks_by_source.assert_not_called() + mock_svc.store.search.assert_called_once() + def test_numeric_ref_with_only_substring_hit_stays_topical(self, mock_svc): """ "12" inside "notes-2012" is a false match; a numeric reference that token-matches nothing must not route via the substring fallback.""" From b59482b328a086d40a95d896369a68840180cbca Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 10 Jul 2026 18:21:44 -0400 Subject: [PATCH 53/77] retrieval: budget context against the engine's served window, not the target The known-item route pulls a whole document, and the context budget trimmed it against the configured context target; the server can come up smaller (the fleet divides context across slots), so the routed document overflowed the real window and the request hard-failed with an HTTP 400. The budget now uses the provider's served per-slot context when known, still capped by an explicit num_ctx, falling back to the configured target when no engine is running. The topical path never exposed this because eight chunks cannot overflow any plausible window. --- src/lilbee/retrieval/query/searcher.py | 10 ++++++- tests/test_query.py | 36 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index 3d8674900..d408f9d50 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -685,8 +685,16 @@ def _fit_context_budget( ``max_context_sources`` caps by count; this caps by tokens so a retrieval-heavy query degrades gracefully instead of erroring with CONTEXT_OVERFLOW. The top-ranked source is always kept. + + The ceiling is the engine's ACTUAL per-slot window when known: the + configured value is a target the dynamic picker aims for, and the + server can come up smaller (the fleet divides context across slots). + Budgeting against the target let a routed whole document overflow + the real window and hard-fail the request with an HTTP 400. """ - ctx = self._config.num_ctx or self._config.chat_n_ctx_target + configured = self._config.num_ctx or self._config.chat_n_ctx_target + served = self._provider.served_chat_ctx() + ctx = min(configured, served) if served else configured reserve = ( estimate_text_tokens(system) + estimate_text_tokens(question) diff --git a/tests/test_query.py b/tests/test_query.py index 6638c29ec..fd7f8fd8d 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1581,6 +1581,42 @@ def test_quoted_title_resolves_via_unique_substring_match(self, mock_svc): assert rag is not None mock_svc.store.get_chunks_by_source.assert_called_once_with("harbor-survey-2010.pdf") + def test_routed_document_fits_the_served_context_window(self, mock_svc): + """The campaign's overflow failure: the route pulls a whole document, + and the budget must trim to the engine's ACTUAL serving window, not + the configured target, or /api/ask hard-fails with a 400.""" + mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] + # A document far larger than the serving window: 200 chunks x ~400 tokens. + mock_svc.store.get_chunks_by_source.return_value = [ + _make_result(source="survey_report.pdf", chunk="word " * 400, chunk_index=i) + for i in range(200) + ] + mock_svc.provider.served_chat_ctx.return_value = 8192 + cfg.num_ctx = None + rag = get_services().searcher.build_rag_context("summarize survey_report.pdf") + assert rag is not None + results, messages = rag + prompt_tokens = sum(len(m["content"]) // 4 for m in messages) + assert prompt_tokens <= 8192 + # Document order preserved after trimming: the head survives. + assert [r.chunk_index for r in results] == list(range(len(results))) + + def test_budget_falls_back_to_config_when_served_ctx_unknown(self, mock_svc): + mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] + mock_svc.store.get_chunks_by_source.return_value = [ + _make_result(source="survey_report.pdf", chunk="word " * 400, chunk_index=i) + for i in range(200) + ] + mock_svc.provider.served_chat_ctx.return_value = None + cfg.num_ctx = 4096 + try: + rag = get_services().searcher.build_rag_context("summarize survey_report.pdf") + finally: + cfg.num_ctx = None + assert rag is not None + _, messages = rag + assert sum(len(m["content"]) // 4 for m in messages) <= 4096 + def test_docket_reference_resolves_by_content_concentration(self, mock_svc): """A reference that appears in a document's text but not its filename (a docket number) resolves when BM25 hits concentrate in one source.""" From e2722f85a45d53f64db49ece4bf493e6e06c12d0 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:48:53 -0400 Subject: [PATCH 54/77] retrieval: opt-in typed entity extraction with schema induction and exact typed counts Count questions over typed records (how many distinct X, how many X per Y) were declined because the store held no typed data: concepts are untyped strings and a general NER tag set has no notion of the identifier types a specific corpus carries. This adds the designed two-phase mode, default off. Phase 1 (lilbee entities induce, cheap): an LLM reads a stratified sample of the indexed chunks and proposes the corpus-specific type schema, written to a reviewable entity_schema.json; the artifact is the contract and is inspected before anything expensive runs. Phase 2 (lilbee entities backfill for the existing index, plus sync-time extraction behind the entity_extraction flag): each type is found by the cheapest extractor that serves it: compiled regex for identifier-shaped types, spaCy labels when the graph extra's model is available, an LLM only for the remainder, with extractor kinds degrading independently. Backfill reads chunk text from the store, so no re-ingest and no re-embedding. The entities table (entity, type, normalized_value, source, page, chunk_index, confidence) is additive: no existing schema changes, no migration, and stores without it read as nothing extracted. Source replacement deletes entity rows with the source's chunks. Values normalize for grouping (casefold, leading-zero-insensitive numerics). Aggregate routing answers the new shapes exactly and without a model: distinct counts scan per type, association counts group by chunk co-occurrence, and questions whose nouns don't resolve against the reviewed schema decline while naming which types are countable. Also fixes the overflow the private-corpus retest isolated: context budgeting now assumes dense-text tokenization (3 chars/token; 4 let a document whose true cost exceeded the window pass untrimmed), and a typed provider context-overflow triggers one refit-and-retry instead of failing the question. --- docs/architecture.md | 31 +++ docs/usage.md | 1 + src/lilbee/app/settings_map.py | 6 + src/lilbee/cli/commands/__init__.py | 2 + src/lilbee/cli/commands/entities.py | 151 ++++++++++++ src/lilbee/core/config/__init__.py | 2 + src/lilbee/core/config/defaults.py | 1 + src/lilbee/core/config/model.py | 5 + src/lilbee/data/ingest/pipeline.py | 64 ++++- src/lilbee/data/ingest/types.py | 5 +- src/lilbee/data/store/core.py | 70 ++++++ src/lilbee/retrieval/entities/__init__.py | 27 +++ src/lilbee/retrieval/entities/extractor.py | 246 ++++++++++++++++++++ src/lilbee/retrieval/entities/schema.py | 120 ++++++++++ src/lilbee/retrieval/query/intent.py | 38 ++- src/lilbee/retrieval/query/searcher.py | 129 ++++++++++- tests/test_entities_cli.py | 206 +++++++++++++++++ tests/test_entities_extractor.py | 257 +++++++++++++++++++++ tests/test_entities_ingest.py | 170 ++++++++++++++ tests/test_entities_store.py | 119 ++++++++++ tests/test_intent.py | 24 +- tests/test_query.py | 123 +++++++++- 22 files changed, 1776 insertions(+), 21 deletions(-) create mode 100644 src/lilbee/cli/commands/entities.py create mode 100644 src/lilbee/retrieval/entities/__init__.py create mode 100644 src/lilbee/retrieval/entities/extractor.py create mode 100644 src/lilbee/retrieval/entities/schema.py create mode 100644 tests/test_entities_cli.py create mode 100644 tests/test_entities_extractor.py create mode 100644 tests/test_entities_ingest.py create mode 100644 tests/test_entities_store.py diff --git a/docs/architecture.md b/docs/architecture.md index 208821217..76c88f6e6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -696,6 +696,37 @@ Two query-time effects: - **Default max sources**: 8 chunks (`LILBEE_MAX_CONTEXT_SOURCES`) - **When it helps**: multi-faceted queries like "compare X and Y" where top-k might only cover X but context selection ensures Y is also represented. +#### Typed Entity Extraction (opt-in ingest mode) +**Off by default** (`LILBEE_ENTITY_EXTRACTION`). Extracts typed entities into a +dedicated `entities` table so count and cross-reference questions get exact +answers from a scan instead of a hedge from top-k retrieval. Additive: the new +table changes no existing schema, requires no migration, and a store without +it behaves as if nothing was extracted. + +Two phases, deliberately operator-driven: + +1. **Induction** (`lilbee entities induce`, cheap): an LLM reads a stratified + sample of the indexed chunks and proposes the corpus-specific type schema, + written to a reviewable `entity_schema.json`. A general NER tag set has no + notion of the identifier types a specific corpus carries; the schema is + the contract, and it is inspected (and edited) before anything expensive + runs. +2. **Application** (`lilbee entities backfill` for the existing index, plus + sync-time extraction for new files once the flag is on): each type is + found by the cheapest extractor that serves it: compiled regex for + identifier-shaped types, spaCy labels for people/organizations/dates + (when the `graph` extra's model is available), and an LLM only for types + neither can catch. Cost is dominated by how many LLM-kind types the + reviewed schema keeps. Backfill reads chunk text from the store, so no + documents are re-ingested and no embeddings are recomputed. + +Query-time effects: "how many distinct X" answers with exact distinct counts, +and "how many X per Y" groups by chunk co-occurrence, both computed by full +scans over the entities table with no LLM in the loop. Count questions whose +nouns don't resolve against the schema decline precisely, naming the types +that are countable. Re-ingesting a source replaces its entity rows the same +way it replaces its chunks. + #### Context Provenance Headers **Always on.** Each context block opens with a one-line header naming its source and page or line span (`[3] (survey_report.pdf, pages 3-4)`), so the answering model can attribute claims to a named document, notice two chunks share a source, and confirm it is reading the document the user asked about. Citation markers in answers are parsed as `[1]`, `[1, 2]`, and bounded ranges like `[1-3]`. diff --git a/docs/usage.md b/docs/usage.md index 5c9b7d164..34efbc44c 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -784,6 +784,7 @@ reason the defaults are the defaults. | `LILBEE_EXPANSION_SIMILARITY_THRESHOLD` | `0.5` | Minimum query-variant cosine similarity to survive the guardrail | | `LILBEE_CANDIDATE_MULTIPLIER` | `3` | Per-arm retrieval depth as a multiple of top_k (hybrid overfetch; also the vector-only MMR pool) | | `LILBEE_FUSION_OVERFETCH_FLOOR` | `50` | Minimum rows fetched per retrieval arm before fusion, regardless of top_k | +| `LILBEE_ENTITY_EXTRACTION` | `false` | Extract typed entities at ingest for exact count answers (needs a reviewed `entity_schema.json`; see `lilbee entities`) | | `LILBEE_FUSION_ALPHA` | `0.6` | Vector-arm weight in hybrid score fusion; the BM25 arm gets the complement | | `LILBEE_MIN_RELEVANCE_SCORE` | `0.0` | Abstention floor against the canonical [0, 1] relevance score; when every result falls below it, ask refuses instead of answering from noise | | `LILBEE_HISTORY_REWRITE` | `true` | Condense follow-up questions into standalone retrieval queries using chat history | diff --git a/src/lilbee/app/settings_map.py b/src/lilbee/app/settings_map.py index 2827e8fff..5edc22298 100644 --- a/src/lilbee/app/settings_map.py +++ b/src/lilbee/app/settings_map.py @@ -143,6 +143,12 @@ def get_default(key: str) -> object: group=SettingGroup.INGEST, help_text="Run a sync before `lilbee ask` (disable on large static corpora)", ), + "entity_extraction": SettingDef( + bool, + nullable=False, + group=SettingGroup.INGEST, + help_text="Extract typed entities at ingest (needs a reviewed entity_schema.json)", + ), "semantic_chunking": SettingDef( bool, nullable=False, diff --git a/src/lilbee/cli/commands/__init__.py b/src/lilbee/cli/commands/__init__.py index df1c4c69c..68c906e74 100644 --- a/src/lilbee/cli/commands/__init__.py +++ b/src/lilbee/cli/commands/__init__.py @@ -12,6 +12,7 @@ from lilbee.cli.commands import ( agent_config, dataset, + entities, ingest_sync, memory, meta, @@ -31,6 +32,7 @@ app.command()(ingest_sync.add) app.command()(ingest_sync.chunks) app.command()(ingest_sync.remove) +app.command()(entities.entities) app.command(name="export")(dataset.export_cmd) app.command(name="import")(dataset.import_cmd) app.command()(search_chat.ask) diff --git a/src/lilbee/cli/commands/entities.py b/src/lilbee/cli/commands/entities.py new file mode 100644 index 000000000..eab2b7b37 --- /dev/null +++ b/src/lilbee/cli/commands/entities.py @@ -0,0 +1,151 @@ +"""The ``lilbee entities`` command: schema induction, backfill, status. + +The mode is deliberately two-phase and operator-driven: ``induce`` samples +the indexed chunks and writes a reviewable ``entity_schema.json``; after the +schema is inspected (and edited if needed), ``backfill`` applies it across +every stored chunk without re-ingesting documents, and syncs keep new files +current once ``entity_extraction`` is enabled. +""" + +from __future__ import annotations + +from pathlib import Path + +import typer + +from lilbee.app.services import get_services +from lilbee.cli.app import apply_overrides, console, data_dir_option, global_option +from lilbee.cli.helpers import json_output +from lilbee.core.config import CHUNKS_TABLE, cfg + +# Stratified induction sample: chunks are read evenly across the table so a +# corpus with several document families contributes all of them. +_INDUCTION_SAMPLE = 40 +# Chunks per extraction batch during backfill; bounds the working set. +_BACKFILL_BATCH = 2000 + + +def _sample_chunks(limit: int) -> list[str]: + store = get_services().store + table = store.open_table(CHUNKS_TABLE) + if table is None: + return [] + total = table.count_rows() + if total == 0: + return [] + step = max(1, total // limit) + texts: list[str] = [] + arrow = table.to_arrow().select(["chunk"]) + index = 0 + for batch in arrow.to_batches(max_chunksize=_BACKFILL_BATCH): + for text in batch.column("chunk").to_pylist(): + if index % step == 0 and len(texts) < limit: + texts.append(text) + index += 1 + return texts + + +def entities( + action: str = typer.Argument(help="induce | backfill | status"), + data_dir: Path | None = data_dir_option, + use_global: bool = global_option, +) -> None: + """Typed entity extraction: propose a schema, backfill the index, or inspect state. + + ``induce`` writes entity_schema.json next to the index for review; + ``backfill`` extracts entities from every stored chunk under the reviewed + schema (no document re-ingest); ``status`` shows the schema and row counts. + """ + apply_overrides(data_dir=data_dir, use_global=use_global) + if action == "induce": + _induce() + elif action == "backfill": + _backfill() + elif action == "status": + _status() + else: + console.print(f"Unknown action {action!r}: expected induce, backfill, or status") + raise SystemExit(2) + + +def _induce() -> None: + from lilbee.retrieval.entities import induce_schema, save_schema + + texts = _sample_chunks(_INDUCTION_SAMPLE) + if not texts: + console.print("Nothing indexed yet; sync documents before inducing a schema.") + raise SystemExit(1) + schema = induce_schema(texts, get_services().provider) + if schema is None: + console.print("Schema induction produced nothing usable; check the chat model and retry.") + raise SystemExit(1) + path = save_schema(schema, cfg.data_dir) + if cfg.json_mode: + json_output({"command": "entities", "action": "induce", "schema": str(path)}) + return + console.print(f"Proposed {len(schema.types)} types; review and edit {path}") + for entity_type in schema.types: + console.print(f" {entity_type.name} ({entity_type.kind.value})") + console.print("Then run: lilbee entities backfill") + + +def _backfill() -> None: + from lilbee.retrieval.concepts import concepts_available + from lilbee.retrieval.entities import ExtractorKind, extract_entities, load_schema + + schema = load_schema(cfg.data_dir) + if schema is None: + console.print("No reviewed entity_schema.json found; run: lilbee entities induce") + raise SystemExit(1) + store = get_services().store + table = store.open_table(CHUNKS_TABLE) + if table is None: + console.print("Nothing indexed yet; sync documents first.") + raise SystemExit(1) + nlp = None + if any(t.kind is ExtractorKind.SPACY for t in schema.types) and concepts_available(): + from lilbee.retrieval.concepts.nlp import _ensure_spacy_model + + nlp = _ensure_spacy_model() + provider = None + if any(t.kind is ExtractorKind.LLM for t in schema.types): + provider = get_services().provider + written = 0 + columns = ["chunk", "source", "chunk_index", "page_start"] + arrow = table.to_arrow().select(columns) + for batch in arrow.to_batches(max_chunksize=_BACKFILL_BATCH): + records = batch.to_pylist() + rows = extract_entities(records, schema, provider=provider, nlp=nlp) + written += store.add_entities(rows) + if cfg.json_mode: + json_output({"command": "entities", "action": "backfill", "rows": written}) + return + console.print(f"Backfill complete: {written} entity rows extracted.") + + +def _status() -> None: + from lilbee.core.config import ENTITIES_TABLE + from lilbee.retrieval.entities import load_schema, schema_path + + schema = load_schema(cfg.data_dir) + table = get_services().store.open_table(ENTITIES_TABLE) + rows = table.count_rows() if table is not None else 0 + if cfg.json_mode: + json_output( + { + "command": "entities", + "action": "status", + "schema": str(schema_path(cfg.data_dir)) if schema else None, + "types": [t.name for t in schema.types] if schema else [], + "rows": rows, + "extraction_enabled": cfg.entity_extraction, + } + ) + return + if schema is None: + console.print("No entity schema; run: lilbee entities induce") + else: + names = ", ".join(t.name for t in schema.types) + console.print(f"Schema: {schema_path(cfg.data_dir)} ({names})") + console.print(f"Extracted rows: {rows}") + console.print(f"Sync-time extraction: {'on' if cfg.entity_extraction else 'off'}") diff --git a/src/lilbee/core/config/__init__.py b/src/lilbee/core/config/__init__.py index 81accf3b0..2b0b1238a 100644 --- a/src/lilbee/core/config/__init__.py +++ b/src/lilbee/core/config/__init__.py @@ -7,6 +7,7 @@ # ruff: noqa: I001 from .defaults import ( CHUNK_CONCEPTS_TABLE as CHUNK_CONCEPTS_TABLE, + ENTITIES_TABLE as ENTITIES_TABLE, CHUNKS_TABLE as CHUNKS_TABLE, CITATIONS_TABLE as CITATIONS_TABLE, CONCEPT_EDGES_TABLE as CONCEPT_EDGES_TABLE, @@ -47,6 +48,7 @@ "DEFAULT_HTTP_TIMEOUT", "DEFAULT_IGNORE_DIRS", "DEFAULT_NUM_CTX", + "ENTITIES_TABLE", "MEMORIES_TABLE", "META_TABLE", "PAGE_TEXTS_TABLE", diff --git a/src/lilbee/core/config/defaults.py b/src/lilbee/core/config/defaults.py index 298d06c72..83eefe51a 100644 --- a/src/lilbee/core/config/defaults.py +++ b/src/lilbee/core/config/defaults.py @@ -45,6 +45,7 @@ CONCEPT_NODES_TABLE = "concept_nodes" CONCEPT_EDGES_TABLE = "concept_edges" CHUNK_CONCEPTS_TABLE = "chunk_concepts" +ENTITIES_TABLE = "entities" # Default URL-exclusion regexes for recursive crawls. Grouped by source # CMS / category. User overrides come from LILBEE_CRAWL_EXCLUDE_PATTERNS diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index 1e108ebce..f24b8a554 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -136,6 +136,11 @@ class Config(BaseSettings): # Tesseract fallback wall-clock timeout per file, seconds. 0 = no cap. tesseract_timeout: float = ConfigField(default=60.0, ge=0.0, writable=True) + # Opt-in typed entity extraction at ingest (an entities table for exact + # counting and cross-referencing). Needs a reviewed entity_schema.json in + # data_dir; without one, syncs skip extraction. Off by default: the + # corpus-scale pass costs real compute and most vaults never need it. + entity_extraction: bool = ConfigField(default=False, writable=True) semantic_chunking: bool = ConfigField(default=False, writable=True) topic_threshold: float = ConfigField(default=0.75, ge=0.0, le=1.0, writable=True) server_host: str = "127.0.0.1" diff --git a/src/lilbee/data/ingest/pipeline.py b/src/lilbee/data/ingest/pipeline.py index d547aea5a..41a45a928 100644 --- a/src/lilbee/data/ingest/pipeline.py +++ b/src/lilbee/data/ingest/pipeline.py @@ -7,7 +7,7 @@ import logging import threading import time -from collections.abc import Callable, Coroutine, Iterable, Iterator +from collections.abc import Callable, Coroutine, Iterable, Iterator, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any, cast @@ -123,6 +123,49 @@ async def _rebuild_concept_clusters() -> None: log.warning("Concept cluster rebuild failed", exc_info=True) +async def _build_entity_records( + records: list[ChunkRecord], source_name: str +) -> list[dict] | None: + """Extract typed entities for ingested chunks. None when the mode is off. + + Gated twice: the ``entity_extraction`` config flag, and the presence of a + reviewed schema artifact; absent either, syncs cost nothing. Extraction + failures degrade to no rows for the file, mirroring concept extraction. + """ + config = active_config() + if not config.entity_extraction or not records: + return None + from lilbee.retrieval.entities import ExtractorKind, extract_entities, load_schema + + schema = load_schema(config.data_dir) + if schema is None: + return None + nlp = None + if any(t.kind is ExtractorKind.SPACY for t in schema.types): + from lilbee.retrieval.concepts import concepts_available + from lilbee.retrieval.concepts.nlp import _ensure_spacy_model + + if concepts_available(): + try: + nlp = _ensure_spacy_model() + except ImportError: + log.warning("spaCy model unavailable; spacy-kind entity types skipped") + provider = None + if any(t.kind is ExtractorKind.LLM for t in schema.types): + provider = get_services().provider + try: + return await to_ingest_thread( + extract_entities, + cast("list[Mapping[str, Any]]", records), + schema, + provider=provider, + nlp=nlp, + ) + except Exception: + log.warning("Entity extraction failed for %s", source_name, exc_info=True) + return None + + async def _build_concept_records( records: list[ChunkRecord], source_name: str ) -> ConceptRecords | None: @@ -641,6 +684,7 @@ async def _process_one(entry: FileToProcess, file_index: int) -> _IngestResult: page_texts_out=page_texts, ) concept_records = await _build_concept_records(records, name) + entity_rows = await _build_entity_records(records, name) on_progress( EventType.FILE_DONE, FileDoneEvent(file=name, status="ok", chunks=len(records)), @@ -657,6 +701,7 @@ async def _process_one(entry: FileToProcess, file_index: int) -> _IngestResult: page_texts=page_texts, stat=entry.stat, concept_records=concept_records, + entity_rows=entity_rows, ) except (asyncio.CancelledError, TaskCancelledError) as exc: # TaskCancelledError is the TUI's cooperative cancel signal raised @@ -986,6 +1031,7 @@ def _flush_batch(buffer: list[_IngestResult]) -> None: ] _retry_after_lock_timeout(lambda: store.write_chunks_batch(items)) _flush_concept_records(buffer) + _flush_entity_rows(buffer) def _flush_concept_records(buffer: list[_IngestResult]) -> None: @@ -1005,6 +1051,22 @@ def _flush_concept_records(buffer: list[_IngestResult]) -> None: log.warning("Concept indexing failed for %d-file batch", len(batches), exc_info=True) +def _flush_entity_rows(buffer: list[_IngestResult]) -> None: + """Write the flush unit's buffered entity rows in one batched pass. + + Runs after the chunk write, which also performed the per-source deletes, + so replacement never leaves a source's stale entity rows behind. A write + failure is logged and never fails the files, matching concept semantics. + """ + rows = [row for r in buffer if r.entity_rows for row in r.entity_rows] + if not rows: + return + try: + get_services().store.add_entities(rows) + except Exception: + log.warning("Entity indexing failed for a %d-row batch", len(rows), exc_info=True) + + def _purge_emptied_sources(names: list[str]) -> None: """Remove the prior index entry for files that now extract to nothing. diff --git a/src/lilbee/data/ingest/types.py b/src/lilbee/data/ingest/types.py index 79bddf7ec..111269790 100644 --- a/src/lilbee/data/ingest/types.py +++ b/src/lilbee/data/ingest/types.py @@ -123,8 +123,8 @@ class _IngestResult: batched flush writes them; ``None`` on a failed file. ``needs_cleanup`` travels with the records so the flush can delete the source's old chunks in the same transaction. ``page_texts`` carries the per-page text dataset rows - and ``concept_records`` the file's concept-table rows, both written by the - same flush. + and ``concept_records`` the file's concept-table rows, and ``entity_rows`` + the file's typed-entity rows, all written by the same flush. """ name: str @@ -137,6 +137,7 @@ class _IngestResult: page_texts: list[PageTextRecord] | None = None stat: SourceStat | None = None concept_records: ConceptRecords | None = None + entity_rows: list[dict] | None = None # Extension → content_type string for document formats handled by kreuzberg. diff --git a/src/lilbee/data/store/core.py b/src/lilbee/data/store/core.py index 43d18b624..7ac11f847 100644 --- a/src/lilbee/data/store/core.py +++ b/src/lilbee/data/store/core.py @@ -17,6 +17,7 @@ CHUNK_CONCEPTS_TABLE, CHUNKS_TABLE, CITATIONS_TABLE, + ENTITIES_TABLE, MEMORIES_TABLE, META_TABLE, PAGE_TEXTS_TABLE, @@ -115,6 +116,7 @@ def _drop_unsupported_far_rows( (CHUNKS_TABLE, "source"), (PAGE_TEXTS_TABLE, "source"), (CHUNK_CONCEPTS_TABLE, "chunk_source"), + (ENTITIES_TABLE, "source"), ) # Stat backfills replace this many source rows per locked write: the first @@ -690,6 +692,74 @@ def _fixed_filter(self, results: list[SearchChunk], threshold: float) -> list[Se """Simple fixed threshold filter - keep only results within distance threshold.""" return [r for r in results if _get_distance(r) <= threshold] + def add_entities(self, records: list[dict]) -> int: + """Append typed entity rows; creates the table on first write. + + Additive to the store: existing tables and schemas are untouched, and + stores without this table behave as if nothing was ever extracted. + """ + if not records: + return 0 + from lilbee.retrieval.entities.schema import _entities_schema + + with self._write_lock(): + db = self.get_db() + table = ensure_table(db, ENTITIES_TABLE, _entities_schema()) + table.add(records) + return len(records) + + def entity_value_counts(self, entity_type: str) -> tuple[int, int]: + """(mentions, distinct normalized values) for one entity type. + + Full scan by design: a count is a corpus property. Streaming batches + keep memory flat at any corpus size. + """ + table = self.open_table(ENTITIES_TABLE) + if table is None: + return 0, 0 + mentions = 0 + values: set[str] = set() + arrow = table.to_arrow().select(["type", "normalized_value"]) + for batch in arrow.to_batches(max_chunksize=_TERM_SCAN_BATCH_ROWS): + types = batch.column("type").to_pylist() + vals = batch.column("normalized_value").to_pylist() + for t_, v in zip(types, vals, strict=True): + if t_ == entity_type: + mentions += 1 + values.add(v) + return mentions, len(values) + + def entity_association_counts(self, counted: str, grouped_by: str) -> dict[str, int]: + """Distinct *counted*-type values co-occurring with each *grouped_by* value. + + Co-occurrence is per chunk: two entities extracted from the same + ``(source, chunk_index)`` are associated. This is the GROUP BY that + answers "how many X is each Y associated with". + """ + table = self.open_table(ENTITIES_TABLE) + if table is None: + return {} + per_chunk: dict[tuple[str, int], tuple[set[str], set[str]]] = {} + arrow = table.to_arrow().select(["type", "normalized_value", "source", "chunk_index"]) + for batch in arrow.to_batches(max_chunksize=_TERM_SCAN_BATCH_ROWS): + rows = zip( + batch.column("type").to_pylist(), + batch.column("normalized_value").to_pylist(), + batch.column("source").to_pylist(), + batch.column("chunk_index").to_pylist(), + strict=True, + ) + for t_, v, src, idx in rows: + if t_ not in (counted, grouped_by): + continue + counted_vals, group_vals = per_chunk.setdefault((src, idx), (set(), set())) + (counted_vals if t_ == counted else group_vals).add(v) + associations: dict[str, set[str]] = {} + for counted_vals, group_vals in per_chunk.values(): + for group_value in group_vals: + associations.setdefault(group_value, set()).update(counted_vals) + return {k: len(v) for k, v in sorted(associations.items())} + def count_term_mentions(self, term: str) -> tuple[int, int]: """(matching chunks, distinct matching sources) for a case-insensitive substring scan of the WHOLE chunks table. diff --git a/src/lilbee/retrieval/entities/__init__.py b/src/lilbee/retrieval/entities/__init__.py new file mode 100644 index 000000000..a2ac56f59 --- /dev/null +++ b/src/lilbee/retrieval/entities/__init__.py @@ -0,0 +1,27 @@ +"""Opt-in typed entity extraction: schema induction, extraction, storage.""" + +from lilbee.retrieval.entities.extractor import ( + extract_entities, + induce_schema, + normalize_value, +) +from lilbee.retrieval.entities.schema import ( + EntitySchema, + EntityType, + ExtractorKind, + load_schema, + save_schema, + schema_path, +) + +__all__ = [ + "EntitySchema", + "EntityType", + "ExtractorKind", + "extract_entities", + "induce_schema", + "load_schema", + "normalize_value", + "save_schema", + "schema_path", +] diff --git a/src/lilbee/retrieval/entities/extractor.py b/src/lilbee/retrieval/entities/extractor.py new file mode 100644 index 000000000..8bfd125f3 --- /dev/null +++ b/src/lilbee/retrieval/entities/extractor.py @@ -0,0 +1,246 @@ +"""Two-phase typed entity extraction. + +Phase 1 (cheap): an LLM reads a stratified sample of chunks and proposes the +corpus-specific type schema, persisted as a reviewable artifact before any +expensive pass runs. + +Phase 2 (scales with corpus): each type is found by the cheapest extractor +that can serve it: compiled regex for identifier-shaped types, spaCy labels +for the general ones, and an LLM only for types neither can catch. Cost is +therefore dominated by how many LLM-kind types the reviewed schema keeps. +""" + +from __future__ import annotations + +import json +import logging +import re +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any + +from lilbee.retrieval.entities.schema import EntitySchema, EntityType, ExtractorKind +from lilbee.retrieval.reasoning import strip_reasoning + +if TYPE_CHECKING: + from lilbee.providers.base import LLMProvider + +log = logging.getLogger(__name__) + +INDUCTION_SAMPLE_SIZE = 40 +INDUCTION_MAX_TOKENS = 1200 +# Chunks per LLM call in phase 2; larger batches save round-trips, smaller +# ones keep each response comfortably parseable. +LLM_EXTRACTION_BATCH = 8 +LLM_EXTRACTION_MAX_TOKENS = 800 + +_CONFIDENCE = {ExtractorKind.REGEX: 1.0, ExtractorKind.SPACY: 0.8, ExtractorKind.LLM: 0.6} + +INDUCTION_PROMPT = ( + "You are designing an entity-extraction schema for a document collection. " + "Below are sample passages. Propose the 3-8 entity TYPES most useful for " + "counting and cross-referencing in this collection.\n" + "Return ONLY a JSON object of the form:\n" + '{{"types": [{{"name": "snake_case_name", "kind": "regex|spacy|llm", ' + '"pattern": "", "description": "one line", ' + '"synonyms": ["words a question would use"]}}]}}\n' + "Prefer regex for identifier-shaped types (codes, numbered records), spacy " + "for people/organizations/dates, llm only when unavoidable.\n\n" + "Passages:\n{sample}" +) + +LLM_EXTRACTION_PROMPT = ( + "Extract entities of these types from each numbered passage.\n" + "Types:\n{types}\n" + "Return ONLY a JSON object mapping passage number to a list of " + '{{"type": ..., "text": ...}} objects; use an empty list when a passage ' + "has none.\n\nPassages:\n{passages}" +) + + +def _first_json_object(text: str) -> dict | None: + """The first balanced JSON object in *text*, or None.""" + start = text.find("{") + if start < 0: + return None + depth = 0 + for i, ch in enumerate(text[start:], start): + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + try: + parsed = json.loads(text[start : i + 1]) + except json.JSONDecodeError: + return None + return parsed if isinstance(parsed, dict) else None + return None + + +def normalize_value(text: str) -> str: + """Canonical form for grouping: casefold, collapse spaces, and strip + leading zeros from purely numeric values so 00482 and 482 group together.""" + value = " ".join(text.split()).casefold() + if value.isdigit(): + value = value.lstrip("0") or "0" + return value + + +def induce_schema(sample_texts: list[str], provider: LLMProvider) -> EntitySchema | None: + """Phase 1: propose a schema from sampled chunk texts. None on failure.""" + if not sample_texts: + return None + sample = "\n---\n".join(t[:600] for t in sample_texts[:INDUCTION_SAMPLE_SIZE]) + prompt = INDUCTION_PROMPT.format(sample=sample) + try: + response = provider.chat( + [{"role": "user", "content": prompt}], + stream=False, + options={"num_predict": INDUCTION_MAX_TOKENS}, + ) + except Exception: + log.warning("Entity schema induction failed at the provider", exc_info=True) + return None + payload = _first_json_object(strip_reasoning(response.text)) + if payload is None: + log.warning("Entity schema induction returned no parseable JSON") + return None + types: list[EntityType] = [] + for raw in payload.get("types", []): + try: + entity_type = EntityType.model_validate(raw) + except Exception: + log.warning("Dropping invalid induced type: %r", raw) + continue + if entity_type.kind is ExtractorKind.REGEX: + try: + re.compile(entity_type.pattern) + except re.error: + log.warning("Dropping induced type %s: bad regex", entity_type.name) + continue + types.append(entity_type) + return EntitySchema(types=types) if types else None + + +def _extract_regex(entity_type: EntityType, text: str) -> list[str]: + return [m.group(0) for m in re.finditer(entity_type.pattern, text)] + + +def _extract_spacy(types: list[EntityType], text: str, nlp: Any) -> list[tuple[EntityType, str]]: + wanted = {t.pattern.upper(): t for t in types} + doc = nlp(text) + found: list[tuple[EntityType, str]] = [] + for ent in getattr(doc, "ents", []): + entity_type = wanted.get(ent.label_) + if entity_type is not None: + found.append((entity_type, ent.text)) + return found + + +def _extract_llm_batch( + types: list[EntityType], + texts: list[str], + provider: LLMProvider, +) -> list[list[tuple[EntityType, str]]]: + by_name = {t.name: t for t in types} + type_lines = "\n".join(f"- {t.name}: {t.description or t.name}" for t in types) + passages = "\n".join(f"[{i}] {t[:800]}" for i, t in enumerate(texts)) + prompt = LLM_EXTRACTION_PROMPT.format(types=type_lines, passages=passages) + empty: list[list[tuple[EntityType, str]]] = [[] for _ in texts] + try: + response = provider.chat( + [{"role": "user", "content": prompt}], + stream=False, + options={"num_predict": LLM_EXTRACTION_MAX_TOKENS}, + ) + except Exception: + log.warning("LLM entity extraction failed for a batch", exc_info=True) + return empty + payload = _first_json_object(strip_reasoning(response.text)) + if payload is None: + return empty + results = empty + for key, items in payload.items(): + try: + index = int(key) + except (TypeError, ValueError): + continue + if not (0 <= index < len(texts)) or not isinstance(items, list): + continue + for item in items: + if not isinstance(item, dict): + continue + entity_type = by_name.get(str(item.get("type", ""))) + mention = str(item.get("text", "")).strip() + if entity_type is not None and mention: + results[index].append((entity_type, mention)) + return results + + +def extract_entities( + chunks: list[Mapping[str, Any]], + schema: EntitySchema, + *, + provider: LLMProvider | None = None, + nlp: Any = None, +) -> list[dict]: + """Phase 2 over ingest-shaped chunk records; returns entities-table rows. + + Each chunk dict needs ``chunk`` (text), ``source``, ``chunk_index``, and + ``page_start``. Extractor kinds degrade independently: regex always runs, + spaCy kinds are skipped without a loaded model, LLM kinds without a + provider, so a partial toolchain yields partial extraction, never failure. + """ + regex_types = [t for t in schema.types if t.kind is ExtractorKind.REGEX] + spacy_types = [t for t in schema.types if t.kind is ExtractorKind.SPACY] + llm_types = [t for t in schema.types if t.kind is ExtractorKind.LLM] + + per_chunk: list[list[tuple[EntityType, str]]] = [] + for record in chunks: + text = record["chunk"] + found: list[tuple[EntityType, str]] = [] + for entity_type in regex_types: + found.extend((entity_type, m) for m in _extract_regex(entity_type, text)) + if spacy_types and nlp is not None: + found.extend(_extract_spacy(spacy_types, text, nlp)) + per_chunk.append(found) + + if llm_types and provider is not None: + for start in range(0, len(chunks), LLM_EXTRACTION_BATCH): + batch = chunks[start : start + LLM_EXTRACTION_BATCH] + batch_found = _extract_llm_batch(llm_types, [r["chunk"] for r in batch], provider) + for offset, found in enumerate(batch_found): + per_chunk[start + offset].extend(found) + + return [ + row + for record, found in zip(chunks, per_chunk, strict=True) + for row in _rows_for_chunk(record, found) + ] + + +def _rows_for_chunk(record: Mapping[str, Any], found: list[tuple[EntityType, str]]) -> list[dict]: + """Deduplicated entities-table rows for one chunk's findings.""" + rows: list[dict] = [] + seen: set[tuple[str, str]] = set() + for entity_type, mention in found: + normalized = normalize_value(mention) + if not normalized: + continue + key = (entity_type.name, normalized) + if key in seen: + continue + seen.add(key) + rows.append( + { + "entity": mention, + "type": entity_type.name, + "normalized_value": normalized, + "source": record["source"], + "page": int(record.get("page_start") or 0), + "chunk_index": int(record["chunk_index"]), + "confidence": _CONFIDENCE[entity_type.kind], + } + ) + return rows diff --git a/src/lilbee/retrieval/entities/schema.py b/src/lilbee/retrieval/entities/schema.py new file mode 100644 index 000000000..83a72743b --- /dev/null +++ b/src/lilbee/retrieval/entities/schema.py @@ -0,0 +1,120 @@ +"""Typed entity extraction: the schema artifact and the entities table. + +The extraction taxonomy is induced from the corpus, not fixed: a general NER +tag set has no notion of the identifier types a specific corpus carries, so a +schema built from a sample is proposed first, written to a reviewable JSON +artifact, and only then applied at scale. The artifact is the contract: a +human (or agent) can edit types, patterns, and synonyms before paying for the +corpus-wide pass. +""" + +from __future__ import annotations + +import json +import logging +from enum import Enum +from pathlib import Path + +import pyarrow as pa +from pydantic import BaseModel, Field, field_validator + +log = logging.getLogger(__name__) + +SCHEMA_FILENAME = "entity_schema.json" + + +class ExtractorKind(Enum): + """How a type's mentions are found, cheapest first.""" + + REGEX = "regex" + SPACY = "spacy" + LLM = "llm" + + +class EntityType(BaseModel): + """One induced type: how to find it and what to call it in questions.""" + + name: str = Field(min_length=1, max_length=64) + kind: ExtractorKind + # REGEX kinds compile this; SPACY kinds name a spaCy label (PERSON, ORG, + # DATE, ...); LLM kinds carry a one-line description for the prompt. + pattern: str = "" + description: str = "" + # Question nouns that mean this type ("part number", "part numbers"). + synonyms: list[str] = Field(default_factory=list) + + @field_validator("name") + @classmethod + def _slugify(cls, v: str) -> str: + slug = "_".join(v.strip().lower().split()) + if not slug.replace("_", "").isalnum(): + raise ValueError(f"type name must be alphanumeric words: {v!r}") + return slug + + +class EntitySchema(BaseModel): + """The reviewable extraction contract for one corpus.""" + + types: list[EntityType] + + def type_for_noun(self, noun: str) -> EntityType | None: + """Resolve a question noun (singular or plural) to a type, if any.""" + wanted = noun.strip().lower() + candidates = {wanted} + if wanted.endswith("s"): + candidates.add(wanted[:-1]) + else: + candidates.add(wanted + "s") + for entity_type in self.types: + names = {entity_type.name, entity_type.name.replace("_", " ")} + names.update(s.strip().lower() for s in entity_type.synonyms) + expanded = set(names) + for n in names: + expanded.add(n + "s" if not n.endswith("s") else n[:-1]) + if candidates & expanded: + return entity_type + return None + + +def schema_path(data_dir: Path) -> Path: + """Where the corpus's schema artifact lives.""" + return data_dir / SCHEMA_FILENAME + + +def load_schema(data_dir: Path) -> EntitySchema | None: + """Read the schema artifact, or ``None`` when absent or unreadable. + + Unreadable is logged, not raised: a hand-edited artifact with a typo + should degrade to "extraction off" rather than break sync. + """ + path = schema_path(data_dir) + if not path.is_file(): + return None + try: + return EntitySchema.model_validate_json(path.read_text()) + except Exception: + log.warning("Entity schema at %s is unreadable; extraction skipped", path, exc_info=True) + return None + + +def save_schema(schema: EntitySchema, data_dir: Path) -> Path: + """Write the schema artifact (pretty, stable order) and return its path.""" + path = schema_path(data_dir) + path.parent.mkdir(parents=True, exist_ok=True) + payload = schema.model_dump(mode="json") + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + return path + + +def _entities_schema(dim_unused: int | None = None) -> pa.Schema: + return pa.schema( + [ + pa.field("entity", pa.utf8()), + pa.field("type", pa.utf8()), + pa.field("normalized_value", pa.utf8()), + pa.field("source", pa.utf8()), + pa.field("page", pa.int32()), + pa.field("chunk_index", pa.int32()), + pa.field("confidence", pa.float32()), + ] + ) diff --git a/src/lilbee/retrieval/query/intent.py b/src/lilbee/retrieval/query/intent.py index 9d0e202ea..169e177b9 100644 --- a/src/lilbee/retrieval/query/intent.py +++ b/src/lilbee/retrieval/query/intent.py @@ -27,15 +27,25 @@ class AggregateKind(Enum): TOTAL_SOURCES = "total_sources" TERM_MENTIONS = "term_mentions" + DISTINCT_TYPE = "distinct_type" + TYPE_ASSOCIATION = "type_association" UNSUPPORTED = "unsupported" @dataclass(frozen=True) class AggregateQuery: - """A parsed aggregate question.""" + """A parsed aggregate question. + + ``noun`` carries the thing being counted for the typed kinds; + ``group_noun`` the per-group dimension of an association question. Both + are question words, resolved against the extraction schema by the caller + (the parser stays schema-free and purely syntactic). + """ kind: AggregateKind term: str = "" + noun: str = "" + group_noun: str = "" # Filename-shaped tokens: a path-ish word with a known document extension. @@ -73,6 +83,22 @@ class AggregateQuery: re.IGNORECASE, ) +# "how many X is each Y associated with" / "how many X per Y": typed +# association counts over extracted entities. +_ASSOCIATION_RE = re.compile( + r"how\s+many\s+(.+?)\s+(?:is|are)\s+each\s+(.+?)\s+" + r"(?:associated\s+with|linked\s+to|recorded\s+(?:for|against))", + re.IGNORECASE, +) +_PER_RE = re.compile(r"how\s+many\s+(.+?)\s+per\s+(.+?)[?.\s]*$", re.IGNORECASE) + +# "how many distinct/unique X ...": typed distinct counts. +_DISTINCT_RE = re.compile( + r"how\s+many\s+(?:distinct|unique|different)\s+(.+?)" + r"(?:\s+(?:are|were|is|exist)\b.*)?[?.\s]*$", + re.IGNORECASE, +) + # "how many documents mention/contain/reference X": term-mention counts. _TERM_MENTION_RE = re.compile( r"how\s+many\s+(?:documents|sources|files|pages|chunks|passages)\s+" @@ -145,6 +171,16 @@ def parse_aggregate(question: str) -> AggregateQuery | None: """ if not _HOW_MANY_RE.search(question): return None + m = _ASSOCIATION_RE.search(question) or _PER_RE.search(question) + if m: + return AggregateQuery( + AggregateKind.TYPE_ASSOCIATION, + noun=m.group(1).strip(), + group_noun=m.group(2).strip(), + ) + m = _DISTINCT_RE.search(question) + if m: + return AggregateQuery(AggregateKind.DISTINCT_TYPE, noun=m.group(1).strip()) m = _TERM_MENTION_RE.search(question) if m: term = m.group(1).strip().strip("\"'") diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index d408f9d50..ee32e7e01 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -22,7 +22,7 @@ cosine_sim, human_recall_predicate, ) -from lilbee.providers.base import LLMProvider +from lilbee.providers.base import LLMProvider, ProviderError, ProviderErrorKind from lilbee.retrieval.embedder import Embedder from lilbee.retrieval.query.dedup import ( _greedy_cover, @@ -45,7 +45,6 @@ cited_subset, strip_llm_citations, ) -from lilbee.retrieval.query.history_window import estimate_text_tokens from lilbee.retrieval.query.intent import ( AggregateKind, AggregateQuery, @@ -135,6 +134,16 @@ def _bm25_confidence(score: float | None) -> float: # Token reserve for the generated answer when budgeting RAG context to num_ctx. _ANSWER_RESERVE_TOKENS = 1024 +# Chars-per-token assumed when BUDGETING context. Deliberately harsher than +# the display estimator's 4: dense OCR/legal text tokenizes at ~2.5-3 chars +# per token, and budgeting at 4 let a document whose real cost exceeded the +# window pass untrimmed, hard-failing the request. +_BUDGET_CHARS_PER_TOKEN = 3 +# Association answers list at most this many groups before summarizing. +_ASSOCIATION_LINES = 15 +# One retry after a provider context-overflow, refitting to this fraction of +# the budget. The estimator is a heuristic; overflow must degrade, not fail. +_OVERFLOW_RETRY_SCALE = 0.6 # Approximate token cost of the Context/Question template wrapper. _CONTEXT_TEMPLATE_TOKENS = 16 # Approximate per-source overhead: the "[i] " marker, the provenance header @@ -663,8 +672,22 @@ def build_rag_context( results = self._reranker.rerank(retrieval_query, results) # Temporal filtering already ran inside search(); no need to repeat it here. results = self.select_context(results, retrieval_query) + return self._finalize_context(results, question, history) + + def _finalize_context( + self, + results: list[SearchChunk], + question: str, + history: list[ChatMessage] | None, + scale: float = 1.0, + ) -> tuple[list[SearchChunk], list[ChatMessage]]: + """Fit *results* to the context budget and assemble the prompt. + + Split from build_rag_context so an overflow retry can refit the same + retrieved set tighter without re-running retrieval or condensation. + """ system = self._system_with_memory(self._config.rag_system_prompt, question) - results = self._fit_context_budget(results, system, question, history) + results = self._fit_context_budget(results, system, question, history, scale) context = build_context(results) prompt = CONTEXT_TEMPLATE.format(context=context, question=question) messages: list[ChatMessage] = [{"role": "system", "content": system}] @@ -673,12 +696,18 @@ def build_rag_context( messages.append({"role": "user", "content": prompt}) return results, messages + @staticmethod + def _budget_tokens(text: str) -> int: + """Conservative token cost for budgeting (see _BUDGET_CHARS_PER_TOKEN).""" + return max(1, len(text) // _BUDGET_CHARS_PER_TOKEN) + def _fit_context_budget( self, results: list[SearchChunk], system: str, question: str, history: list[ChatMessage] | None, + scale: float = 1.0, ) -> list[SearchChunk]: """Drop the lowest-ranked sources until the assembled prompt fits num_ctx. @@ -696,17 +725,17 @@ def _fit_context_budget( served = self._provider.served_chat_ctx() ctx = min(configured, served) if served else configured reserve = ( - estimate_text_tokens(system) - + estimate_text_tokens(question) - + sum(estimate_text_tokens(m["content"]) for m in history or []) + self._budget_tokens(system) + + self._budget_tokens(question) + + sum(self._budget_tokens(m["content"]) for m in history or []) + _ANSWER_RESERVE_TOKENS + _CONTEXT_TEMPLATE_TOKENS ) - budget = ctx - reserve + budget = int((ctx - reserve) * scale) kept: list[SearchChunk] = [] used = 0 for r in results: - cost = estimate_text_tokens(r.chunk) + _PER_SOURCE_TOKENS + cost = self._budget_tokens(r.chunk) + _PER_SOURCE_TOKENS if kept and used + cost > budget: break kept.append(r) @@ -770,6 +799,24 @@ def _answer_aggregate(self, aggregate: AggregateQuery) -> str: f"{aggregate.term!r}, across {chunk_hits} passages. This counts literal " f"mentions of the phrase, not paraphrases." ) + if aggregate.kind in (AggregateKind.DISTINCT_TYPE, AggregateKind.TYPE_ASSOCIATION): + typed = self._answer_typed_aggregate(aggregate) + if typed is not None: + return typed + return self._decline_aggregate() + + def _decline_aggregate(self) -> str: + """The honest no-capability answer, naming what IS countable.""" + from lilbee.retrieval.entities import load_schema + + schema = load_schema(self._config.data_dir) + if schema is not None and schema.types: + countable = ", ".join(sorted(t.name.replace("_", " ") for t in schema.types)) + return ( + "That count isn't answerable from the extracted records. Countable " + f"entity types in this index: {countable}. I can also count documents " + "or passages that mention a specific term." + ) return ( "Answering that count needs structured records (dates, identifiers, or " "entities) that aren't extracted from this corpus yet. I can count " @@ -777,6 +824,53 @@ def _answer_aggregate(self, aggregate: AggregateQuery) -> str: "the passages themselves and count from those." ) + def _answer_typed_aggregate(self, aggregate: AggregateQuery) -> str | None: + """Exact answers over extracted entities, or None when the question's + nouns don't resolve against the reviewed schema.""" + from lilbee.retrieval.entities import load_schema + + schema = load_schema(self._config.data_dir) + counted = schema.type_for_noun(aggregate.noun) if schema else None + if schema is None or counted is None: + return None + if aggregate.kind is AggregateKind.DISTINCT_TYPE: + return self._answer_distinct_count(counted.name) + grouped = schema.type_for_noun(aggregate.group_noun) + if grouped is None: + return None + return self._answer_association_count(counted.name, grouped.name) + + def _answer_distinct_count(self, type_name: str) -> str: + pretty = type_name.replace("_", " ") + mentions, distinct = self._store.entity_value_counts(type_name) + if mentions == 0: + return ( + f"No {pretty} entities are extracted yet; " + "run a sync with entity extraction enabled first." + ) + return ( + f"Exact scan of the extracted records: {distinct} distinct " + f"{pretty} values, across {mentions} mentions." + ) + + def _answer_association_count(self, counted: str, grouped: str) -> str: + counted_pretty = counted.replace("_", " ") + grouped_pretty = grouped.replace("_", " ") + counts = self._store.entity_association_counts(counted, grouped_by=grouped) + if not counts: + return ( + f"No co-occurring {counted_pretty} and {grouped_pretty} entities are " + "extracted yet; run a sync with entity extraction enabled first." + ) + shown = list(counts.items())[:_ASSOCIATION_LINES] + lines = "\n".join(f" {value}: {n}" for value, n in shown) + more = len(counts) - len(shown) + suffix = f"\n ... and {more} more" if more > 0 else "" + return ( + f"Exact counts from the extracted records ({counted_pretty} " + f"per {grouped_pretty}, by shared passage):\n{lines}{suffix}" + ) + def route_direct_answer(self, question: str) -> str | None: """The exact-scan answer for a count-shaped question, else ``None``. @@ -868,9 +962,24 @@ def ask_raw( if rag is None: return AskResult(answer=_GROUNDED_REFUSAL, sources=[]) results, messages = rag - provider_messages = self._messages_for_provider(messages) opts = options if options is not None else self._config.generation_options() - result = self._provider.chat(provider_messages, options=opts or None) + try: + result = self._provider.chat( + self._messages_for_provider(messages), options=opts or None + ) + except ProviderError as exc: + if exc.kind is not ProviderErrorKind.CONTEXT_OVERFLOW or not results: + raise + # The budget estimator is a heuristic; when the engine still + # reports overflow, refit the same retrieved set tighter and + # retry once instead of hard-failing the question. + log.warning("Context overflow despite budgeting; retrying with a tighter fit") + results, messages = self._finalize_context( + results, question, history, scale=_OVERFLOW_RETRY_SCALE + ) + result = self._provider.chat( + self._messages_for_provider(messages), options=opts or None + ) raw = result.text clean = raw if self._config.show_reasoning else strip_reasoning(raw) return AskResult(answer=clean, sources=results, cited_sources=cited_subset(clean, results)) diff --git a/tests/test_entities_cli.py b/tests/test_entities_cli.py new file mode 100644 index 000000000..95e08ddc1 --- /dev/null +++ b/tests/test_entities_cli.py @@ -0,0 +1,206 @@ +"""End-to-end tests for the ``lilbee entities`` command.""" + +import pytest +from typer.testing import CliRunner + +import lilbee.app.services as svc_mod +import lilbee.cli.commands # noqa: F401 (registers commands on the app) +from lilbee.cli.app import app +from lilbee.core.config import cfg +from lilbee.data.store import Store +from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind, save_schema + +runner = CliRunner() + + +@pytest.fixture() +def isolated(tmp_path, monkeypatch): + """A real Store on a tmp data root, pinned via --data-dir on every invoke. + + The CLI callback re-resolves the data root, so tests pass the flag + explicitly instead of trusting pre-set singleton paths. + """ + from tests.conftest import make_mock_services + + monkeypatch.delenv("LILBEE_DATA", raising=False) + snapshot = cfg.model_copy() + cfg.data_root = tmp_path + cfg.documents_dir = tmp_path / "documents" + cfg.data_dir = tmp_path / "data" + cfg.lancedb_dir = tmp_path / "data" / "lancedb" + cfg.data_dir.mkdir(parents=True, exist_ok=True) + store = Store(cfg) + services = make_mock_services(store=store) + svc_mod.set_services(services) + yield store + svc_mod.set_services(None) + for name in type(cfg).model_fields: + setattr(cfg, name, getattr(snapshot, name)) + + +def _invoke(action, tmp_root): + return runner.invoke(app, ["entities", action, "--data-dir", str(tmp_root)]) + + +def _index_chunks(store, texts): + dim = cfg.embedding_dim + store.add_chunks( + [ + { + "source": "catalog.txt", + "content_type": "text", + "chunk_type": "raw", + "page_start": 1, + "page_end": 1, + "line_start": 0, + "line_end": 0, + "chunk": text, + "chunk_index": i, + "vector": [0.1] * dim, + } + for i, text in enumerate(texts) + ] + ) + + +def _invoke_json(action, tmp_root): + return runner.invoke(app, ["--json", "entities", action, "--data-dir", str(tmp_root)]) + + +class TestEntitiesCommand: + def test_json_status_and_backfill_and_induce(self, isolated): + import json as jsonlib + from types import SimpleNamespace + + _index_chunks(isolated, ["part PX4471"]) + services = svc_mod.get_services() + services.provider.chat.return_value = SimpleNamespace( + text='{"types": [{"name": "part_number", "kind": "regex", ' + '"pattern": "PX\\\\d{4}", "description": "ids", "synonyms": []}]}' + ) + induced = _invoke_json("induce", cfg.data_root) + assert induced.exit_code == 0 + assert jsonlib.loads(induced.output)["action"] == "induce" + backfilled = _invoke_json("backfill", cfg.data_root) + assert jsonlib.loads(backfilled.output)["rows"] == 1 + status = _invoke_json("status", cfg.data_root) + payload = jsonlib.loads(status.output) + assert payload["rows"] == 1 + assert payload["types"] == ["part_number"] + + def test_backfill_with_spacy_and_llm_types(self, isolated): + from types import SimpleNamespace + from unittest import mock as umock + + _index_chunks(isolated, ["part PX4471 shipped"]) + save_schema( + EntitySchema( + types=[ + EntityType(name="person", kind=ExtractorKind.SPACY, pattern="PERSON"), + EntityType(name="vessel", kind=ExtractorKind.LLM, description="ships"), + EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}"), + ] + ), + cfg.data_dir, + ) + services = svc_mod.get_services() + services.provider.chat.return_value = SimpleNamespace(text="{}") + fake_nlp = umock.MagicMock(return_value=umock.MagicMock(ents=[])) + with ( + umock.patch("lilbee.retrieval.concepts.concepts_available", return_value=True), + umock.patch("lilbee.retrieval.concepts.nlp._ensure_spacy_model", return_value=fake_nlp), + ): + result = _invoke("backfill", cfg.data_root) + assert result.exit_code == 0 + assert "1 entity rows" in result.output + + def test_backfill_with_nothing_indexed(self, isolated): + save_schema( + EntitySchema( + types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] + ), + cfg.data_dir, + ) + result = _invoke("backfill", cfg.data_root) + assert result.exit_code == 1 + assert "sync documents" in result.output + + def test_induce_with_emptied_table(self, isolated): + """A chunks table whose rows were all removed samples nothing.""" + _index_chunks(isolated, ["part PX4471"]) + isolated.upsert_source("catalog.txt", "h", chunk_count=1) + isolated.remove_documents(["catalog.txt"]) + result = _invoke("induce", cfg.data_root) + assert result.exit_code == 1 + assert "sync documents" in result.output + + def test_status_before_anything(self, isolated): + result = _invoke("status", cfg.data_root) + assert result.exit_code == 0 + assert "No entity schema" in result.output + + def test_backfill_requires_schema(self, isolated): + result = _invoke("backfill", cfg.data_root) + assert result.exit_code == 1 + assert "induce" in result.output + + def test_backfill_extracts_over_stored_chunks(self, isolated): + """The no-re-ingest path: rows come from the chunks table alone.""" + _index_chunks(isolated, ["part PX4471 shipped", "parts PX9001 and PX4471"]) + save_schema( + EntitySchema( + types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] + ), + cfg.data_dir, + ) + result = _invoke("backfill", cfg.data_root) + assert result.exit_code == 0 + assert "3 entity rows" in result.output + mentions, distinct = isolated.entity_value_counts("part_number") + assert (mentions, distinct) == (3, 2) + + def test_status_after_backfill(self, isolated): + _index_chunks(isolated, ["part PX4471"]) + save_schema( + EntitySchema( + types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] + ), + cfg.data_dir, + ) + _invoke("backfill", cfg.data_root) + result = _invoke("status", cfg.data_root) + assert result.exit_code == 0 + assert "part_number" in result.output + assert "Extracted rows: 1" in result.output + + def test_induce_writes_reviewable_schema(self, isolated, monkeypatch): + from types import SimpleNamespace + + _index_chunks(isolated, ["part PX4471 shipped from the depot"]) + services = svc_mod.get_services() + services.provider.chat.return_value = SimpleNamespace( + text='{"types": [{"name": "part_number", "kind": "regex", ' + '"pattern": "PX\\\\d{4}", "description": "ids", "synonyms": []}]}' + ) + result = _invoke("induce", cfg.data_root) + assert result.exit_code == 0 + assert "review and edit" in result.output + assert (cfg.data_dir / "entity_schema.json").is_file() + + def test_induce_with_unusable_model_output(self, isolated): + from types import SimpleNamespace + + _index_chunks(isolated, ["part PX4471"]) + svc_mod.get_services().provider.chat.return_value = SimpleNamespace(text="no json") + result = _invoke("induce", cfg.data_root) + assert result.exit_code == 1 + assert "nothing usable" in result.output + + def test_unknown_action(self, isolated): + result = _invoke("frobnicate", cfg.data_root) + assert result.exit_code == 2 + + def test_induce_with_empty_index(self, isolated): + result = _invoke("induce", cfg.data_root) + assert result.exit_code == 1 + assert "sync documents" in result.output diff --git a/tests/test_entities_extractor.py b/tests/test_entities_extractor.py new file mode 100644 index 000000000..562ef0eae --- /dev/null +++ b/tests/test_entities_extractor.py @@ -0,0 +1,257 @@ +"""Tests for entity schema induction and two-phase extraction.""" + +import json +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from lilbee.retrieval.entities import ( + EntitySchema, + EntityType, + ExtractorKind, + extract_entities, + induce_schema, + load_schema, + normalize_value, + save_schema, +) + + +def _text_result(text): + return SimpleNamespace(text=text) + + +def _chunk(text, source="a.txt", idx=0, page=1): + return {"chunk": text, "source": source, "chunk_index": idx, "page_start": page} + + +PART = EntityType( + name="part_number", + kind=ExtractorKind.REGEX, + pattern=r"PX\d{4}", + synonyms=["part number"], +) +PERSON = EntityType(name="person", kind=ExtractorKind.SPACY, pattern="PERSON") +VESSEL = EntityType(name="vessel", kind=ExtractorKind.LLM, description="named ships or boats") + + +class TestNormalizeValue: + def test_casefolds_and_collapses(self): + assert normalize_value(" Split Rock ") == "split rock" + + def test_numeric_leading_zeros(self): + assert normalize_value("00482") == "482" + assert normalize_value("0") == "0" + + +class TestSchemaArtifact: + def test_round_trip(self, tmp_path): + schema = EntitySchema(types=[PART, PERSON]) + path = save_schema(schema, tmp_path) + assert path.name == "entity_schema.json" + loaded = load_schema(tmp_path) + assert loaded is not None + assert [t.name for t in loaded.types] == ["part_number", "person"] + + def test_absent_reads_none(self, tmp_path): + assert load_schema(tmp_path) is None + + def test_unreadable_reads_none(self, tmp_path): + (tmp_path / "entity_schema.json").write_text("{not json") + assert load_schema(tmp_path) is None + + def test_type_for_noun_matches_synonyms_and_plurals(self): + schema = EntitySchema(types=[PART]) + assert schema.type_for_noun("part numbers") is not None + assert schema.type_for_noun("Part Number") is not None + assert schema.type_for_noun("vessels") is None + + +class TestInduceSchema: + def test_parses_valid_proposal(self): + provider = MagicMock() + provider.chat.return_value = _text_result( + "identifiers dominate" + + json.dumps( + { + "types": [ + { + "name": "part number", + "kind": "regex", + "pattern": r"PX\d{4}", + "description": "catalog part ids", + "synonyms": ["part"], + }, + {"name": "person", "kind": "spacy", "pattern": "PERSON"}, + ] + } + ) + ) + schema = induce_schema(["part PX4471 shipped"], provider) + assert schema is not None + assert [t.name for t in schema.types] == ["part_number", "person"] + + def test_drops_bad_regex_keeps_rest(self): + provider = MagicMock() + provider.chat.return_value = _text_result( + json.dumps( + { + "types": [ + {"name": "broken", "kind": "regex", "pattern": "(["}, + {"name": "person", "kind": "spacy", "pattern": "PERSON"}, + ] + } + ) + ) + schema = induce_schema(["text"], provider) + assert schema is not None + assert [t.name for t in schema.types] == ["person"] + + def test_unparseable_response_is_none(self): + provider = MagicMock() + provider.chat.return_value = _text_result("no json here") + assert induce_schema(["text"], provider) is None + + def test_provider_error_is_none(self): + provider = MagicMock() + provider.chat.side_effect = RuntimeError("down") + assert induce_schema(["text"], provider) is None + + def test_empty_sample_is_none(self): + assert induce_schema([], MagicMock()) is None + + +class TestFirstJsonObjectEdges: + def test_malformed_json_is_none(self): + from lilbee.retrieval.entities.extractor import _first_json_object + + assert _first_json_object('{"a": }') is None + + def test_unbalanced_is_none(self): + from lilbee.retrieval.entities.extractor import _first_json_object + + assert _first_json_object("{unclosed") is None + + +class TestInduceSchemaEdges: + def test_non_dict_type_entry_dropped(self): + provider = MagicMock() + provider.chat.return_value = _text_result( + json.dumps( + {"types": ["nonsense", {"name": "person", "kind": "spacy", "pattern": "PERSON"}]} + ) + ) + schema = induce_schema(["text"], provider) + assert schema is not None + assert [t.name for t in schema.types] == ["person"] + + def test_invalid_type_name_rejected(self): + with pytest.raises(Exception, match="alphanumeric"): + EntityType(name="bad!name", kind=ExtractorKind.REGEX, pattern="x") + + +class _FakeEnt(SimpleNamespace): + pass + + +class _FakeNlp: + def __call__(self, text): + ents = [] + if "Larsen" in text: + ents.append(_FakeEnt(label_="PERSON", text="E. Larsen")) + return SimpleNamespace(ents=ents) + + +class TestExtractEntities: + def test_regex_kind(self): + rows = extract_entities( + [_chunk("parts PX4471 and PX9001, PX4471 again")], + EntitySchema(types=[PART]), + ) + assert {(r["entity"], r["normalized_value"]) for r in rows} == { + ("PX4471", "px4471"), + ("PX9001", "px9001"), + } + assert all(r["confidence"] == 1.0 for r in rows) + + def test_spacy_kind_uses_injected_nlp(self): + rows = extract_entities( + [_chunk("keeper E. Larsen wrote nightly")], + EntitySchema(types=[PERSON]), + nlp=_FakeNlp(), + ) + assert rows and rows[0]["type"] == "person" + assert rows[0]["confidence"] == pytest.approx(0.8) + + def test_spacy_kind_skipped_without_nlp(self): + rows = extract_entities( + [_chunk("keeper E. Larsen wrote nightly")], + EntitySchema(types=[PERSON]), + nlp=None, + ) + assert rows == [] + + def test_llm_kind_batches_and_parses(self): + provider = MagicMock() + provider.chat.return_value = _text_result( + json.dumps({"0": [{"type": "vessel", "text": "the Meridian"}], "1": []}) + ) + rows = extract_entities( + [_chunk("the Meridian docked"), _chunk("no ships here", idx=1)], + EntitySchema(types=[VESSEL]), + provider=provider, + ) + assert len(rows) == 1 + assert rows[0]["entity"] == "the Meridian" + assert rows[0]["confidence"] == pytest.approx(0.6) + + def test_llm_kind_skipped_without_provider(self): + rows = extract_entities( + [_chunk("the Meridian docked")], EntitySchema(types=[VESSEL]), provider=None + ) + assert rows == [] + + def test_llm_failure_degrades_to_empty(self): + provider = MagicMock() + provider.chat.side_effect = RuntimeError("down") + rows = extract_entities( + [_chunk("the Meridian docked")], EntitySchema(types=[VESSEL]), provider=provider + ) + assert rows == [] + + def test_llm_response_edge_shapes_are_ignored(self): + provider = MagicMock() + provider.chat.return_value = _text_result( + json.dumps( + { + "zz": [{"type": "vessel", "text": "Meridian"}], + "9": [{"type": "vessel", "text": "OutOfRange"}], + "0": "not a list", + "1": [42, {"type": "unknown", "text": "x"}, {"type": "vessel", "text": ""}], + } + ) + ) + rows = extract_entities( + [_chunk("one"), _chunk("two", idx=1)], EntitySchema(types=[VESSEL]), provider=provider + ) + assert rows == [] + + def test_llm_unparseable_response_yields_nothing(self): + provider = MagicMock() + provider.chat.return_value = _text_result("plain prose, no json") + rows = extract_entities([_chunk("one")], EntitySchema(types=[VESSEL]), provider=provider) + assert rows == [] + + def test_whitespace_only_mention_is_dropped(self): + blank = EntityType(name="blank", kind=ExtractorKind.REGEX, pattern=r" {2}") + rows = extract_entities([_chunk("a b")], EntitySchema(types=[blank])) + assert rows == [] + + def test_rows_carry_provenance_and_dedupe_per_chunk(self): + rows = extract_entities( + [_chunk("PX4471 PX4471", source="s.txt", idx=3, page=7)], + EntitySchema(types=[PART]), + ) + assert len(rows) == 1 + assert (rows[0]["source"], rows[0]["chunk_index"], rows[0]["page"]) == ("s.txt", 3, 7) diff --git a/tests/test_entities_ingest.py b/tests/test_entities_ingest.py new file mode 100644 index 000000000..46e472a5e --- /dev/null +++ b/tests/test_entities_ingest.py @@ -0,0 +1,170 @@ +"""Tests for the entity-extraction ingest stage and flush.""" + +import asyncio +from unittest import mock + +import pytest + +import lilbee.app.services as svc_mod +from lilbee.core.config import cfg +from lilbee.data.ingest.pipeline import _build_entity_records, _flush_entity_rows +from lilbee.data.ingest.types import _IngestResult +from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind, save_schema + + +@pytest.fixture() +def isolated_cfg(tmp_path): + snapshot = cfg.model_copy() + cfg.data_dir = tmp_path / "data" + cfg.data_dir.mkdir(parents=True, exist_ok=True) + cfg.entity_extraction = False + yield tmp_path + for name in type(cfg).model_fields: + setattr(cfg, name, getattr(snapshot, name)) + + +@pytest.fixture() +def mock_svc(isolated_cfg): + from tests.conftest import make_mock_services + + services = make_mock_services() + svc_mod.set_services(services) + yield services + svc_mod.set_services(None) + + +def _records(): + return [ + { + "source": "a.txt", + "content_type": "text", + "chunk_type": "raw", + "page_start": 1, + "page_end": 1, + "line_start": 0, + "line_end": 0, + "chunk": "parts PX4471 and PX9001", + "chunk_index": 0, + "vector": [0.1], + } + ] + + +def _part_schema(): + return EntitySchema( + types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] + ) + + +class TestBuildEntityRecords: + def test_gate_off_is_none(self, mock_svc): + assert asyncio.run(_build_entity_records(_records(), "a.txt")) is None + + def test_no_schema_is_none(self, mock_svc): + cfg.entity_extraction = True + assert asyncio.run(_build_entity_records(_records(), "a.txt")) is None + + def test_extracts_rows_with_reviewed_schema(self, mock_svc): + cfg.entity_extraction = True + save_schema(_part_schema(), cfg.data_dir) + rows = asyncio.run(_build_entity_records(_records(), "a.txt")) + assert rows is not None + assert {r["normalized_value"] for r in rows} == {"px4471", "px9001"} + assert all(r["source"] == "a.txt" for r in rows) + + def test_extraction_failure_degrades_to_none(self, mock_svc): + cfg.entity_extraction = True + save_schema(_part_schema(), cfg.data_dir) + with mock.patch( + "lilbee.retrieval.entities.extract_entities", side_effect=RuntimeError("boom") + ): + assert asyncio.run(_build_entity_records(_records(), "a.txt")) is None + + def test_empty_records_is_none(self, mock_svc): + cfg.entity_extraction = True + assert asyncio.run(_build_entity_records([], "a.txt")) is None + + +class TestExtractorToolWiring: + def _schema_with(self, kinds): + types = [] + if "spacy" in kinds: + types.append(EntityType(name="person", kind=ExtractorKind.SPACY, pattern="PERSON")) + if "llm" in kinds: + types.append(EntityType(name="vessel", kind=ExtractorKind.LLM, description="ships")) + types.append(EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")) + return EntitySchema(types=types) + + def test_spacy_types_load_the_model_when_available(self, mock_svc): + cfg.entity_extraction = True + save_schema(self._schema_with({"spacy"}), cfg.data_dir) + fake_nlp = mock.MagicMock(return_value=mock.MagicMock(ents=[])) + with ( + mock.patch("lilbee.retrieval.concepts.concepts_available", return_value=True), + mock.patch("lilbee.retrieval.concepts.nlp._ensure_spacy_model", return_value=fake_nlp), + ): + rows = asyncio.run(_build_entity_records(_records(), "a.txt")) + assert rows is not None + fake_nlp.assert_called() + + def test_spacy_model_import_error_degrades(self, mock_svc, caplog): + cfg.entity_extraction = True + save_schema(self._schema_with({"spacy"}), cfg.data_dir) + with ( + mock.patch("lilbee.retrieval.concepts.concepts_available", return_value=True), + mock.patch( + "lilbee.retrieval.concepts.nlp._ensure_spacy_model", + side_effect=ImportError("no model"), + ), + caplog.at_level("WARNING"), + ): + rows = asyncio.run(_build_entity_records(_records(), "a.txt")) + assert rows is not None # regex type still extracted + assert any("spaCy model unavailable" in r.message for r in caplog.records) + + def test_llm_types_fetch_the_provider(self, mock_svc): + cfg.entity_extraction = True + save_schema(self._schema_with({"llm"}), cfg.data_dir) + mock_svc.provider.chat.return_value = mock.MagicMock(text="{}") + rows = asyncio.run(_build_entity_records(_records(), "a.txt")) + assert rows is not None + mock_svc.provider.chat.assert_called() + + +class TestFlushEntityRows: + def _result(self, rows): + return _IngestResult( + name="a.txt", path=cfg.data_dir / "a.txt", chunk_count=1, error=None, entity_rows=rows + ) + + def test_writes_merged_rows(self, mock_svc): + row = { + "entity": "PX4471", + "type": "part_number", + "normalized_value": "px4471", + "source": "a.txt", + "page": 1, + "chunk_index": 0, + "confidence": 1.0, + } + _flush_entity_rows([self._result([row]), self._result(None)]) + mock_svc.store.add_entities.assert_called_once_with([row]) + + def test_no_rows_no_write(self, mock_svc): + _flush_entity_rows([self._result(None), self._result([])]) + mock_svc.store.add_entities.assert_not_called() + + def test_write_failure_is_logged_not_raised(self, mock_svc, caplog): + mock_svc.store.add_entities.side_effect = RuntimeError("locked") + row = { + "entity": "x", + "type": "t", + "normalized_value": "x", + "source": "a.txt", + "page": 1, + "chunk_index": 0, + "confidence": 1.0, + } + with caplog.at_level("WARNING"): + _flush_entity_rows([self._result([row])]) + assert any("Entity indexing failed" in r.message for r in caplog.records) diff --git a/tests/test_entities_store.py b/tests/test_entities_store.py new file mode 100644 index 000000000..24d9a9514 --- /dev/null +++ b/tests/test_entities_store.py @@ -0,0 +1,119 @@ +"""Tests for the entities table: writes, scans, replacement, compatibility.""" + +import pytest + +from lilbee.core.config import cfg +from lilbee.data.store import Store + + +@pytest.fixture() +def store(tmp_path): + config = cfg.model_copy(update={"lancedb_dir": tmp_path / "lancedb_test"}) + return Store(config) + + +def _entity(entity, type_, value, source, chunk_index=0, page=1): + return { + "entity": entity, + "type": type_, + "normalized_value": value, + "source": source, + "page": page, + "chunk_index": chunk_index, + "confidence": 1.0, + } + + +def _chunk_record(source, idx, text, dim): + return { + "source": source, + "content_type": "text", + "chunk_type": "raw", + "page_start": 0, + "page_end": 0, + "line_start": 0, + "line_end": 0, + "chunk": text, + "chunk_index": idx, + "vector": [0.1] * dim, + } + + +class TestEntityWritesAndScans: + def test_value_counts(self, store): + store.add_entities( + [ + _entity("PX4471", "part_number", "px4471", "a.txt", 0), + _entity("PX4471", "part_number", "px4471", "b.txt", 0), + _entity("PX9001", "part_number", "px9001", "b.txt", 1), + _entity("Fresno", "depot", "fresno", "a.txt", 0), + ] + ) + mentions, distinct = store.entity_value_counts("part_number") + assert (mentions, distinct) == (3, 2) + assert store.entity_value_counts("vessel") == (0, 0) + + def test_empty_write_is_a_noop(self, store): + assert store.add_entities([]) == 0 + + def test_association_scan_skips_unrelated_types(self, store): + store.add_entities( + [ + _entity("S1", "shipment", "s1", "a.txt", 0), + _entity("PX4471", "part_number", "px4471", "a.txt", 0), + _entity("Fresno", "depot", "fresno", "a.txt", 0), + ] + ) + counts = store.entity_association_counts("shipment", grouped_by="part_number") + assert counts == {"px4471": 1} + + def test_association_counts_by_chunk_cooccurrence(self, store): + store.add_entities( + [ + # chunk a.txt#0: shipment S1 with part PX4471 + _entity("S1", "shipment", "s1", "a.txt", 0), + _entity("PX4471", "part_number", "px4471", "a.txt", 0), + # chunk a.txt#1: shipments S2, S3 with part PX4471 + _entity("S2", "shipment", "s2", "a.txt", 1), + _entity("S3", "shipment", "s3", "a.txt", 1), + _entity("PX4471", "part_number", "px4471", "a.txt", 1), + # chunk b.txt#0: shipment S1 again with part PX9001 + _entity("S1", "shipment", "s1", "b.txt", 0), + _entity("PX9001", "part_number", "px9001", "b.txt", 0), + ] + ) + counts = store.entity_association_counts("shipment", grouped_by="part_number") + assert counts == {"px4471": 3, "px9001": 1} + + def test_source_replacement_deletes_entity_rows(self, store): + """Removing a source must delete its entity rows like its chunks.""" + dim = store._config.embedding_dim + store.add_chunks([_chunk_record("a.txt", 0, "part PX4471", dim)]) + store.upsert_source("a.txt", "hash-a", chunk_count=1) + store.add_entities( + [ + _entity("PX4471", "part_number", "px4471", "a.txt", 0), + _entity("PX9001", "part_number", "px9001", "b.txt", 0), + ] + ) + result = store.remove_documents(["a.txt"]) + assert result.removed == ["a.txt"] + # a.txt's entity rows are gone; the other source's row survives. + mentions, distinct = store.entity_value_counts("part_number") + assert (mentions, distinct) == (1, 1) + + +class TestCompatibility: + def test_scans_are_empty_on_store_without_entities_table(self, store): + """A pre-entities store (no table) reads as nothing extracted.""" + assert store.entity_value_counts("anything") == (0, 0) + assert store.entity_association_counts("a", grouped_by="b") == {} + + def test_existing_tables_untouched_by_entity_writes(self, store): + dim = store._config.embedding_dim + store.add_chunks([_chunk_record("a.txt", 0, "hello world", dim)]) + before = store.count_chunks() + store.add_entities([_entity("X1", "part_number", "x1", "a.txt", 0)]) + assert store.count_chunks() == before + results = store.search([0.1] * dim, top_k=1) + assert results and results[0].source == "a.txt" diff --git a/tests/test_intent.py b/tests/test_intent.py index f66c5b0d6..95bde87d2 100644 --- a/tests/test_intent.py +++ b/tests/test_intent.py @@ -75,11 +75,29 @@ def test_total_sources(self): assert agg is not None assert agg.kind is AggregateKind.TOTAL_SOURCES - def test_typed_count_is_unsupported_not_topical(self): - """Counts over records the store does not hold must be declined - precisely, not fed to retrieval that cannot count.""" + def test_association_question_parses_both_nouns(self): agg = parse_aggregate("how many shipments is each part number associated with?") assert agg is not None + assert agg.kind is AggregateKind.TYPE_ASSOCIATION + assert (agg.noun, agg.group_noun) == ("shipments", "part number") + + def test_per_question_parses_both_nouns(self): + agg = parse_aggregate("how many deliveries per depot?") + assert agg is not None + assert agg.kind is AggregateKind.TYPE_ASSOCIATION + assert (agg.noun, agg.group_noun) == ("deliveries", "depot") + + def test_distinct_question_parses_noun(self): + agg = parse_aggregate("how many distinct part numbers are recorded?") + assert agg is not None + assert agg.kind is AggregateKind.DISTINCT_TYPE + assert agg.noun == "part numbers" + + def test_unmatched_typed_count_is_unsupported_not_topical(self): + """Counts the parser cannot shape must be declined precisely, not fed + to retrieval that cannot count.""" + agg = parse_aggregate("how many of the entries were amended twice?") + assert agg is not None assert agg.kind is AggregateKind.UNSUPPORTED diff --git a/tests/test_query.py b/tests/test_query.py index fd7f8fd8d..3bdd202d9 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1601,6 +1601,47 @@ def test_routed_document_fits_the_served_context_window(self, mock_svc): # Document order preserved after trimming: the head survives. assert [r.chunk_index for r in results] == list(range(len(results))) + def test_dense_text_budgets_conservatively(self, mock_svc): + """The private-corpus overflow: text tokenizing at ~2.5 chars/token + passed a chars/4 budget untrimmed. Budgeting must assume dense text, + so the kept set stays well under the window even at 3 chars/token.""" + mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] + # Numeric-dense chunks: ~6000 chars each, real cost ~2400 tokens each. + dense = ("4471 0482 9001 " * 400).strip() + mock_svc.store.get_chunks_by_source.return_value = [ + _make_result(source="survey_report.pdf", chunk=dense, chunk_index=i) for i in range(40) + ] + mock_svc.provider.served_chat_ctx.return_value = 24576 + cfg.num_ctx = None + rag = get_services().searcher.build_rag_context("summarize survey_report.pdf") + assert rag is not None + _, messages = rag + # At 2.56 chars/token the real prompt cost must still fit the window. + real_tokens = sum(len(m["content"]) / 2.56 for m in messages) + assert real_tokens <= 24576 * 1.05 + + def test_overflow_from_provider_retries_once_with_tighter_fit(self, mock_svc): + """When the engine still reports overflow, ask refits and retries + instead of surfacing a hard failure.""" + from lilbee.providers.base import ProviderError, ProviderErrorKind + + mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] + mock_svc.store.get_chunks_by_source.return_value = [ + _make_result(source="survey_report.pdf", chunk="word " * 400, chunk_index=i) + for i in range(40) + ] + mock_svc.provider.served_chat_ctx.return_value = 8192 + mock_svc.provider.chat.side_effect = [ + ProviderError("overflow", kind=ProviderErrorKind.CONTEXT_OVERFLOW), + _text_result("fits now [1]"), + ] + result = get_services().searcher.ask_raw("summarize survey_report.pdf") + assert result.answer.startswith("fits now") + assert mock_svc.provider.chat.call_count == 2 + first = mock_svc.provider.chat.call_args_list[0][0][0][-1]["content"] + second = mock_svc.provider.chat.call_args_list[1][0][0][-1]["content"] + assert len(second) < len(first) + def test_budget_falls_back_to_config_when_served_ctx_unknown(self, mock_svc): mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] mock_svc.store.get_chunks_by_source.return_value = [ @@ -1735,10 +1776,15 @@ def test_total_count(self, mock_svc): assert "369" in result.answer assert "123456" in result.answer - def test_typed_count_declines_precisely(self, mock_svc): - result = get_services().searcher.ask_raw( - "how many shipments is each part number associated with?" - ) + def test_typed_count_declines_precisely(self, mock_svc, tmp_path): + old_dir = cfg.data_dir + cfg.data_dir = tmp_path / "no_schema_here" + try: + result = get_services().searcher.ask_raw( + "how many shipments is each part number associated with?" + ) + finally: + cfg.data_dir = old_dir assert "aren't extracted" in result.answer mock_svc.provider.chat.assert_not_called() @@ -1774,6 +1820,75 @@ def test_disabled_by_config_goes_topical(self, mock_svc): mock_svc.store.search.assert_called_once() +class TestTypedAggregates: + @pytest.fixture() + def part_schema(self, tmp_path): + from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind, save_schema + + schema = EntitySchema( + types=[ + EntityType( + name="part_number", + kind=ExtractorKind.REGEX, + pattern=r"PX\d{4}", + synonyms=["part"], + ), + EntityType(name="depot", kind=ExtractorKind.SPACY, pattern="GPE"), + ] + ) + old_dir = cfg.data_dir + cfg.data_dir = tmp_path + save_schema(schema, tmp_path) + yield schema + cfg.data_dir = old_dir + + def test_distinct_count_answers_exactly(self, mock_svc, part_schema): + mock_svc.store.entity_value_counts.return_value = (57, 12) + result = get_services().searcher.ask_raw("how many distinct part numbers are recorded?") + assert "12 distinct part number values" in result.answer + assert "57 mentions" in result.answer + mock_svc.provider.chat.assert_not_called() + + def test_association_answers_grouped_counts(self, mock_svc, part_schema): + mock_svc.store.entity_association_counts.return_value = {"fresno": 3, "reno": 1} + result = get_services().searcher.ask_raw("how many parts is each depot associated with?") + assert "fresno: 3" in result.answer + assert "reno: 1" in result.answer + mock_svc.store.entity_association_counts.assert_called_once_with( + "part_number", grouped_by="depot" + ) + + def test_unresolvable_noun_declines_naming_types(self, mock_svc, part_schema): + result = get_services().searcher.ask_raw("how many distinct vessels are recorded?") + assert "Countable entity types" in result.answer + assert "part number" in result.answer + + def test_empty_extraction_says_so(self, mock_svc, part_schema): + mock_svc.store.entity_value_counts.return_value = (0, 0) + result = get_services().searcher.ask_raw("how many distinct part numbers are there?") + assert "extracted yet" in result.answer + + def test_association_with_unresolvable_group_noun_declines(self, mock_svc, part_schema): + result = get_services().searcher.ask_raw("how many parts is each vessel associated with?") + assert "Countable entity types" in result.answer + + def test_association_with_no_extracted_rows_says_so(self, mock_svc, part_schema): + mock_svc.store.entity_association_counts.return_value = {} + result = get_services().searcher.ask_raw("how many parts is each depot associated with?") + assert "extracted yet" in result.answer + + def test_without_schema_keeps_the_generic_decline(self, mock_svc, tmp_path): + old_dir = cfg.data_dir + cfg.data_dir = tmp_path / "no_schema_here" + try: + result = get_services().searcher.ask_raw( + "how many parts is each depot associated with?" + ) + finally: + cfg.data_dir = old_dir + assert "aren't extracted from this corpus yet" in result.answer + + class TestHistoryCondensation: _HISTORY: ClassVar[list[dict[str, str]]] = [ {"role": "user", "content": "who kept the lighthouse journal at Split Rock"}, From f3a1daad6382da1ccccda4a7a22c7812a9ce34c4 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:57:58 -0400 Subject: [PATCH 55/77] review: dedupe the induction sample constant, document the entities CLI, drop a defensive getattr Blast-radius findings on the entity mode: the CLI duplicated the extractor's sample-size constant; the CLI commands doc had no entities rows; spaCy docs always carry .ents so the getattr defaulted nothing; and the streaming path's no-retry-on-overflow decision is now stated where it lives. --- docs/usage.md | 3 +++ src/lilbee/cli/commands/entities.py | 8 ++++---- src/lilbee/data/ingest/pipeline.py | 4 +--- src/lilbee/retrieval/entities/extractor.py | 2 +- src/lilbee/retrieval/query/searcher.py | 3 +++ 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 34efbc44c..e6975b64d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -508,6 +508,9 @@ lilbee ask "Explain this" --model qwen3 | `lilbee sync` | Re-index changed files | | `lilbee rebuild` | Nuke the database and re-ingest everything | | `lilbee export pages.parquet` | Write a per-page text dataset (parquet or jsonl, no vectors) | +| `lilbee entities induce` | Propose a typed entity schema from the indexed chunks (writes a reviewable `entity_schema.json`) | +| `lilbee entities backfill` | Extract typed entities from every stored chunk under the reviewed schema (no re-ingest) | +| `lilbee entities status` | Show the entity schema, extracted row count, and whether sync-time extraction is on | | `lilbee import pages.parquet` | Import a dataset, re-embedding it with the current model | | `lilbee reset` | Factory reset. Deletes all documents and data | diff --git a/src/lilbee/cli/commands/entities.py b/src/lilbee/cli/commands/entities.py index eab2b7b37..15d1c1cd1 100644 --- a/src/lilbee/cli/commands/entities.py +++ b/src/lilbee/cli/commands/entities.py @@ -18,14 +18,13 @@ from lilbee.cli.helpers import json_output from lilbee.core.config import CHUNKS_TABLE, cfg -# Stratified induction sample: chunks are read evenly across the table so a -# corpus with several document families contributes all of them. -_INDUCTION_SAMPLE = 40 # Chunks per extraction batch during backfill; bounds the working set. _BACKFILL_BATCH = 2000 def _sample_chunks(limit: int) -> list[str]: + """Stratified sample: chunks read evenly across the table so a corpus with + several document families contributes all of them.""" store = get_services().store table = store.open_table(CHUNKS_TABLE) if table is None: @@ -70,8 +69,9 @@ def entities( def _induce() -> None: from lilbee.retrieval.entities import induce_schema, save_schema + from lilbee.retrieval.entities.extractor import INDUCTION_SAMPLE_SIZE - texts = _sample_chunks(_INDUCTION_SAMPLE) + texts = _sample_chunks(INDUCTION_SAMPLE_SIZE) if not texts: console.print("Nothing indexed yet; sync documents before inducing a schema.") raise SystemExit(1) diff --git a/src/lilbee/data/ingest/pipeline.py b/src/lilbee/data/ingest/pipeline.py index 41a45a928..32675130d 100644 --- a/src/lilbee/data/ingest/pipeline.py +++ b/src/lilbee/data/ingest/pipeline.py @@ -123,9 +123,7 @@ async def _rebuild_concept_clusters() -> None: log.warning("Concept cluster rebuild failed", exc_info=True) -async def _build_entity_records( - records: list[ChunkRecord], source_name: str -) -> list[dict] | None: +async def _build_entity_records(records: list[ChunkRecord], source_name: str) -> list[dict] | None: """Extract typed entities for ingested chunks. None when the mode is off. Gated twice: the ``entity_extraction`` config flag, and the presence of a diff --git a/src/lilbee/retrieval/entities/extractor.py b/src/lilbee/retrieval/entities/extractor.py index 8bfd125f3..7e32d530a 100644 --- a/src/lilbee/retrieval/entities/extractor.py +++ b/src/lilbee/retrieval/entities/extractor.py @@ -131,7 +131,7 @@ def _extract_spacy(types: list[EntityType], text: str, nlp: Any) -> list[tuple[E wanted = {t.pattern.upper(): t for t in types} doc = nlp(text) found: list[tuple[EntityType, str]] = [] - for ent in getattr(doc, "ents", []): + for ent in doc.ents: entity_type = wanted.get(ent.label_) if entity_type is not None: found.append((entity_type, ent.text)) diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index ee32e7e01..478d8c252 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -1053,6 +1053,9 @@ def ask_stream( yield StreamToken(content=_GROUNDED_REFUSAL, is_reasoning=False) return results, messages = rag + # No overflow retry here: a stream cannot be rebuilt once tokens have + # been yielded, so the conservative budget in _fit_context_budget is + # the streaming path's protection. provider_messages = self._messages_for_provider(messages) opts = options if options is not None else self._config.generation_options() answer_parts: list[str] = [] From 835d45ffec166a75c6ea0496d6a194eae448da32 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:01:15 -0400 Subject: [PATCH 56/77] entities: match anchored patterns per token; disclose proxy counts The first real-corpus validation found extraction far too sparse: induced regex patterns anchor with ^...$, which matches only a token that IS the identifier, while identifiers in OCR running text appear inline with adjacent punctuation and were never found. A fully-anchored pattern now full-matches each identifier-shaped token instead of the chunk (partial tokens still rejected; alternations of anchored branches fall through untouched), and the induction prompt forbids anchors outright, so both induced and hand-corrected schemas work as written. Separately, a question noun that reaches a type through a synonym was silently answered with a different quantity (counting tail numbers is not counting flights). Distinct-count answers now state which type they measured and that it may not be the asked-for quantity whenever the noun is not the type's own name. --- src/lilbee/retrieval/entities/extractor.py | 37 ++++++++++++++++++++-- src/lilbee/retrieval/query/searcher.py | 26 +++++++++++++-- tests/test_entities_extractor.py | 34 ++++++++++++++++++++ tests/test_query.py | 13 ++++++++ 4 files changed, 105 insertions(+), 5 deletions(-) diff --git a/src/lilbee/retrieval/entities/extractor.py b/src/lilbee/retrieval/entities/extractor.py index 7e32d530a..e81a7ad1c 100644 --- a/src/lilbee/retrieval/entities/extractor.py +++ b/src/lilbee/retrieval/entities/extractor.py @@ -45,7 +45,9 @@ 'spacy kinds, empty for llm kinds>", "description": "one line", ' '"synonyms": ["words a question would use"]}}]}}\n' "Prefer regex for identifier-shaped types (codes, numbered records), spacy " - "for people/organizations/dates, llm only when unavoidable.\n\n" + "for people/organizations/dates, llm only when unavoidable. Regex patterns " + "must match identifiers INLINE in running text: describe the identifier " + "token itself and never use ^ or $ anchors.\n\n" "Passages:\n{sample}" ) @@ -123,8 +125,39 @@ def induce_schema(sample_texts: list[str], provider: LLMProvider) -> EntitySchem return EntitySchema(types=types) if types else None +# Tokens for anchored-pattern matching: identifier-shaped runs of word chars +# (hyphens allowed inside), so punctuation adjacent to an inline identifier +# never defeats the match. +_IDENTIFIER_TOKEN_RE = re.compile(r"[0-9A-Za-z][0-9A-Za-z-]*") + + def _extract_regex(entity_type: EntityType, text: str) -> list[str]: - return [m.group(0) for m in re.finditer(entity_type.pattern, text)] + """Regex mentions, treating anchored patterns as per-token full matches. + + Schema authors (and the induction model) tend to write ^...$ patterns + that describe an identifier's whole shape. Applied to running text those + match almost nothing, since identifiers appear inline with adjacent + punctuation; a ^...$ pattern therefore full-matches each identifier + token instead of the chunk. + """ + pattern = entity_type.pattern + inner_text = pattern[1:-1] + # Only a single fully-anchored pattern converts; an alternation of + # anchored branches (^A$|^B$) would be mangled by stripping the outer + # pair, so it falls through to finditer unchanged. + if ( + pattern.startswith("^") + and pattern.endswith("$") + and "^" not in inner_text + and "$" not in inner_text + ): + inner = re.compile(inner_text) + return [ + token.group(0) + for token in _IDENTIFIER_TOKEN_RE.finditer(text) + if inner.fullmatch(token.group(0)) + ] + return [m.group(0) for m in re.finditer(pattern, text)] def _extract_spacy(types: list[EntityType], text: str, nlp: Any) -> list[tuple[EntityType, str]]: diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index 478d8c252..c9fc6472d 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -119,6 +119,17 @@ def _bm25_confidence(score: float | None) -> float: return score / (score + _BM25_HALF_SATURATION) +def _noun_names_type(noun: str, type_name: str) -> bool: + """Whether the question's noun IS the type (modulo case/space/plural), + as opposed to reaching it through a synonym.""" + wanted = " ".join(noun.strip().lower().split()) + names = {type_name, type_name.replace("_", " ")} + variants = set(names) + for n in names: + variants.add(n + "s" if not n.endswith("s") else n[:-1]) + return wanted in variants + + # RAG mode answer when retrieval finds no usable sources: a grounded refusal # instead of free-wheeling on the model's parametric knowledge. Users who want # off-corpus answers can switch to chat mode. @@ -834,13 +845,13 @@ def _answer_typed_aggregate(self, aggregate: AggregateQuery) -> str | None: if schema is None or counted is None: return None if aggregate.kind is AggregateKind.DISTINCT_TYPE: - return self._answer_distinct_count(counted.name) + return self._answer_distinct_count(counted.name, asked_for=aggregate.noun) grouped = schema.type_for_noun(aggregate.group_noun) if grouped is None: return None return self._answer_association_count(counted.name, grouped.name) - def _answer_distinct_count(self, type_name: str) -> str: + def _answer_distinct_count(self, type_name: str, asked_for: str = "") -> str: pretty = type_name.replace("_", " ") mentions, distinct = self._store.entity_value_counts(type_name) if mentions == 0: @@ -848,10 +859,19 @@ def _answer_distinct_count(self, type_name: str) -> str: f"No {pretty} entities are extracted yet; " "run a sync with entity extraction enabled first." ) - return ( + answer = ( f"Exact scan of the extracted records: {distinct} distinct " f"{pretty} values, across {mentions} mentions." ) + if asked_for and not _noun_names_type(asked_for, type_name): + # A synonym resolved the question's noun to a proxy type; counting + # one is not counting the other (one aircraft flies many flights), + # so the answer must say which quantity it actually measured. + answer += ( + f" Note: this counts {pretty} values, the closest extracted type to " + f"{asked_for.strip()!r}, which may not be the same quantity." + ) + return answer def _answer_association_count(self, counted: str, grouped: str) -> str: counted_pretty = counted.replace("_", " ") diff --git a/tests/test_entities_extractor.py b/tests/test_entities_extractor.py index 562ef0eae..11975f749 100644 --- a/tests/test_entities_extractor.py +++ b/tests/test_entities_extractor.py @@ -175,6 +175,40 @@ def test_regex_kind(self): } assert all(r["confidence"] == 1.0 for r in rows) + def test_anchored_pattern_matches_identifiers_inline(self): + """The OCR-corpus failure: an induced pattern anchored with ^...$ + matches only a token that IS the identifier, so identifiers inline in + running text (with adjacent punctuation) were never found. Anchored + patterns must apply per token.""" + anchored = EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"^PX\d{4}$") + rows = extract_entities( + [_chunk("shipment via PX4471, then PX9001; see also (PX4471).")], + EntitySchema(types=[anchored]), + ) + assert {r["normalized_value"] for r in rows} == {"px4471", "px9001"} + + def test_anchored_pattern_still_rejects_partial_tokens(self): + anchored = EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"^PX\d{4}$") + rows = extract_entities( + [_chunk("codes PX44712 and XPX4471 are different families")], + EntitySchema(types=[anchored]), + ) + assert rows == [] + + def test_anchored_alternation_is_not_mangled(self): + """^A$|^B$ must not have its outer anchors stripped (that would turn + it into A$|^B); it falls through to plain finditer untouched.""" + alternation = EntityType( + name="part_number", kind=ExtractorKind.REGEX, pattern=r"^PX\d{4}$|^QX\d{4}$" + ) + rows = extract_entities( + [_chunk("PX4471"), _chunk("inline PX4471 here", idx=1)], + EntitySchema(types=[alternation]), + ) + # Whole-token chunk still matches via finditer; inline stays unmatched, + # same as before the per-token conversion existed. + assert [r["chunk_index"] for r in rows] == [0] + def test_spacy_kind_uses_injected_nlp(self): rows = extract_entities( [_chunk("keeper E. Larsen wrote nightly")], diff --git a/tests/test_query.py b/tests/test_query.py index 3bdd202d9..731cad4e8 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1868,6 +1868,19 @@ def test_empty_extraction_says_so(self, mock_svc, part_schema): result = get_services().searcher.ask_raw("how many distinct part numbers are there?") assert "extracted yet" in result.answer + def test_synonym_proxy_count_discloses_the_measured_type(self, mock_svc, part_schema): + """A noun resolved through a synonym gets the count of a DIFFERENT + quantity; the answer must say which type it actually measured.""" + mock_svc.store.entity_value_counts.return_value = (57, 12) + result = get_services().searcher.ask_raw("how many distinct parts are recorded?") + assert "12 distinct part number values" in result.answer + assert "closest extracted type" in result.answer + + def test_direct_type_name_carries_no_proxy_note(self, mock_svc, part_schema): + mock_svc.store.entity_value_counts.return_value = (57, 12) + result = get_services().searcher.ask_raw("how many distinct part numbers are recorded?") + assert "closest extracted type" not in result.answer + def test_association_with_unresolvable_group_noun_declines(self, mock_svc, part_schema): result = get_services().searcher.ask_raw("how many parts is each vessel associated with?") assert "Countable entity types" in result.answer From dc444ee3e6268603f84a9332842dd2c090e22634 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 10 Jul 2026 23:19:39 -0400 Subject: [PATCH 57/77] entities: backfill --replace clears prior rows first Backfill appends (sync owns per-source replacement), so re-running it after a pattern fix double-counts every entity. The flag clears the table before extracting, which is the natural workflow when iterating on a schema against an existing index. --- docs/usage.md | 2 +- src/lilbee/cli/commands/entities.py | 13 +++++++++++-- tests/test_entities_cli.py | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index e6975b64d..dfea3b398 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -509,7 +509,7 @@ lilbee ask "Explain this" --model qwen3 | `lilbee rebuild` | Nuke the database and re-ingest everything | | `lilbee export pages.parquet` | Write a per-page text dataset (parquet or jsonl, no vectors) | | `lilbee entities induce` | Propose a typed entity schema from the indexed chunks (writes a reviewable `entity_schema.json`) | -| `lilbee entities backfill` | Extract typed entities from every stored chunk under the reviewed schema (no re-ingest) | +| `lilbee entities backfill` | Extract typed entities from every stored chunk under the reviewed schema (no re-ingest; `--replace` clears prior rows first) | | `lilbee entities status` | Show the entity schema, extracted row count, and whether sync-time extraction is on | | `lilbee import pages.parquet` | Import a dataset, re-embedding it with the current model | | `lilbee reset` | Factory reset. Deletes all documents and data | diff --git a/src/lilbee/cli/commands/entities.py b/src/lilbee/cli/commands/entities.py index 15d1c1cd1..39ad113cf 100644 --- a/src/lilbee/cli/commands/entities.py +++ b/src/lilbee/cli/commands/entities.py @@ -46,6 +46,12 @@ def _sample_chunks(limit: int) -> list[str]: def entities( action: str = typer.Argument(help="induce | backfill | status"), + replace: bool = typer.Option( + False, + "--replace", + help="backfill only: clear previously extracted rows first, so repeated " + "backfills (after a pattern fix) don't double-count", + ), data_dir: Path | None = data_dir_option, use_global: bool = global_option, ) -> None: @@ -59,7 +65,7 @@ def entities( if action == "induce": _induce() elif action == "backfill": - _backfill() + _backfill(replace=replace) elif action == "status": _status() else: @@ -89,7 +95,8 @@ def _induce() -> None: console.print("Then run: lilbee entities backfill") -def _backfill() -> None: +def _backfill(*, replace: bool = False) -> None: + from lilbee.core.config import ENTITIES_TABLE from lilbee.retrieval.concepts import concepts_available from lilbee.retrieval.entities import ExtractorKind, extract_entities, load_schema @@ -102,6 +109,8 @@ def _backfill() -> None: if table is None: console.print("Nothing indexed yet; sync documents first.") raise SystemExit(1) + if replace: + store.clear_table(ENTITIES_TABLE, "entity IS NOT NULL") nlp = None if any(t.kind is ExtractorKind.SPACY for t in schema.types) and concepts_available(): from lilbee.retrieval.concepts.nlp import _ensure_spacy_model diff --git a/tests/test_entities_cli.py b/tests/test_entities_cli.py index 95e08ddc1..ef8a26807 100644 --- a/tests/test_entities_cli.py +++ b/tests/test_entities_cli.py @@ -114,6 +114,24 @@ def test_backfill_with_spacy_and_llm_types(self, isolated): assert result.exit_code == 0 assert "1 entity rows" in result.output + def test_backfill_replace_clears_previous_rows(self, isolated): + """A repeated backfill after a pattern fix must not double-count.""" + _index_chunks(isolated, ["part PX4471"]) + save_schema( + EntitySchema( + types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] + ), + cfg.data_dir, + ) + _invoke("backfill", cfg.data_root) + _invoke("backfill", cfg.data_root) + assert isolated.entity_value_counts("part_number")[0] == 2 # appended: doubled + result = runner.invoke( + app, ["entities", "backfill", "--replace", "--data-dir", str(cfg.data_root)] + ) + assert result.exit_code == 0 + assert isolated.entity_value_counts("part_number") == (1, 1) + def test_backfill_with_nothing_indexed(self, isolated): save_schema( EntitySchema( From d57fe981ab68892819df25a11ef652254b4b7298 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:19:26 -0400 Subject: [PATCH 58/77] searcher: log expansion skips and variant counts at info The evaluation campaign observed fully deterministic retrieval with expansion nominally enabled and could not tell whether expansion ran; the skip decision and variant count were silent. Both now log at info so a single run answers it. --- src/lilbee/retrieval/query/searcher.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index c9fc6472d..398be7e24 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -268,7 +268,9 @@ def _llm_expand(self, question: str, count: int) -> list[str]: ) text = strip_reasoning(response.text).strip() variants = [_strip_list_marker(line.strip()) for line in text.split("\n") if line.strip()] - return [v for v in variants if v][:count] + kept = [v for v in variants if v][:count] + log.info("Query expansion produced %d variants", len(kept)) + return kept def _expand_query( self, question: str, question_vec: list[float] @@ -327,7 +329,14 @@ def _should_skip_expansion(self, question: str, chunk_type: ChunkType | None = N # can never fire exactly when the lexical arm is most certain. second_raw = results[1].bm25_score or 0.0 relative_gap = (top_raw - second_raw) / top_raw if top_raw > 0 else 0.0 - return relative_gap >= self._config.expansion_skip_gap + skip = relative_gap >= self._config.expansion_skip_gap + if skip: + log.info( + "Query expansion skipped: BM25 confident (raw %.1f, gap %.0f%%)", + top_raw, + relative_gap * 100, + ) + return skip def _apply_concept_boost(self, results: list[SearchChunk], question: str) -> list[SearchChunk]: if not self._config.concept_graph or not results: From f370db7d5cc2e544b94e2c4ce3440844705087f4 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:23:53 -0400 Subject: [PATCH 59/77] fix(retrieval): return hybrid fusion to reciprocal rank; no MMR on the hybrid path Graded relevance evaluation (LLM-judged precision@20, equal k, same judge and index across arms) showed the convex score fusion regressing raw hybrid precision about 20% against the old RRF ordering, at every blend weight, so the regression was structural rather than tunable. Two mechanisms, both reproduced in synthetic tests: - MMR newly ran on the hybrid path (it never did before this branch) and at the default lambda it diversifies away exactly what a lexical query's relevant set looks like: many mutually similar passages quoting the same identifiers. Removing it also exposed that the convex formula never actually guaranteed lexical-only survival; MMR had been rescuing the invariant test by accident. - Convex combination of normalized raw scores gives every dense neighbor a score floor (cosine similarities sit in a high narrow band) that crowds out lexically-certain rows the vector arm never saw. fuse_arms is now equal-weight reciprocal rank fusion expressed as a canonical [0,1] score (mean of per-arm (K+1)/(K+rank), K=60), and the arms stay exactly top_k deep: deep pools flood rank fusion with both-arm mediocre rows and bury single-arm certainty, which the invariant test demonstrates. fusion_alpha and fusion_overfetch_floor were knobs of the convex formula and are removed with it; both were born on this branch. Everything else the branch added stands: ANN recall recovery on the hybrid vector arm, canonical score downstream, dedup keeping both provenance fields, the far-row drop that spares BM25-supported rows, intent routing, and context budgeting. --- docs/architecture.md | 17 ++-- docs/usage.md | 4 +- src/lilbee/app/settings_map.py | 12 --- src/lilbee/core/config/model.py | 17 +--- src/lilbee/data/store/core.py | 36 ++++----- src/lilbee/data/store/fusion.py | 65 +++++++++------- src/lilbee/data/store/ranking.py | 12 +-- tests/test_store.py | 9 ++- tests/test_store_fusion.py | 129 +++++++++++++++++++++++-------- 9 files changed, 168 insertions(+), 133 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 76c88f6e6..f47f920d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -588,23 +588,24 @@ flowchart TD #### History Condensation **On by default** (`LILBEE_HISTORY_REWRITE`). A follow-up question is condensed into a standalone retrieval query using the recent chat history (one small LLM call, skipped when there is no history). Retrieval sees only the query text: without this, "what about his brother?" is embedded and BM25-matched with its pronouns. The user's original wording still reaches the answering prompt. -#### Hybrid Search (Dual-Arm Score Fusion) -**Always on.** Retrieves the BM25 arm (LanceDB FTS) and the vector arm independently, each overfetched well past `top_k` (`candidate_multiplier × top_k`, floored at `fusion_overfetch_floor`, default 50 rows per arm), then fuses by convex combination into one canonical [0, 1] score: +#### Hybrid Search (Dual-Arm Reciprocal-Rank Fusion) +**Always on.** Retrieves the BM25 arm (LanceDB FTS) and the vector arm independently, each fetched exactly `top_k` deep, then fuses by reciprocal rank into one canonical [0, 1] score: ``` -score = fusion_alpha × vector_similarity + (1 - fusion_alpha) × normalized_bm25 +score = (rank_weight(vector_rank) + rank_weight(bm25_rank)) / 2, rank_weight(r) = 61 / (60 + r) ``` -Vector similarity is clamped `1 - cosine_distance` (an absolute signal); BM25 is normalized against the top score of its list. `fusion_alpha` defaults to 0.6. +A row ranked first by both arms scores 1.0; a row one arm never retrieved contributes 0 for that arm, so an arm's top hit still scores 0.5 and stays visible next to rows deep in the other arm. The fused ordering is final: no diversity selection runs on the hybrid path. -- **Why score fusion and not rank fusion**: Reciprocal Rank Fusion (the previous mechanism) discards score magnitude by construction, so it cannot tell an arm that was certain from an arm that was guessing. One strong lexical hit for an identifier query drowned under mediocre dense neighbors, fetched from a fusion pool that was only `top_k` deep per arm. Score-aware fusion keeps the certainty, and the overfetch makes rows ranked just past `top_k` in both arms visible to fusion at all. (On the rank-vs-score question, see Bruch et al. 2024, "[An Analysis of Fusion Functions for Hybrid Retrieval](https://dl.acm.org/doi/10.1145/3596512)".) -- **Canonical score**: every search path sets `SearchChunk.score`, and every downstream stage (sorting, filtering, MMR, greedy set cover, concept boost, reranker blending) compares only that field. `distance`, `bm25_score`, and the legacy RRF `relevance_score` remain as provenance. -- **Abstention**: because the canonical score is [0, 1] with real meaning, `min_relevance_score` is a usable floor: when every retrieved chunk falls below it, ask refuses instead of feeding noise as context. RRF magnitudes (~0.016-0.033 total range) made any threshold meaningless. +- **Why rank fusion and not score fusion**: a convex combination of normalized raw scores (`alpha × vector_similarity + (1 − alpha) × normalized_bm25`) was tried here and regressed graded precision about 20% against RRF, at every blend weight. Cosine similarities sit in a high narrow band, giving every dense neighbor a score floor that crowds out lexically-certain rows; ranks are scale-free, so neither arm's score distribution can drown the other. (On the rank-vs-score question, see Bruch et al. 2024, "[An Analysis of Fusion Functions for Hybrid Retrieval](https://dl.acm.org/doi/10.1145/3596512)".) Arm depth matters as much as the formula: a row one arm is certain about scores a fixed 0.5, while rows both arms rank mid-pool accumulate two contributions, so deep candidate pools flood the fused top-k with both-arm mediocrity; arms therefore stay `top_k` deep. +- **Why no MMR on this path**: for lexical queries the relevant passages are often mutually similar (they quote the same identifiers), which is exactly what diversity selection penalizes; running MMR over the fused pool measurably traded relevant lexical hits for diverse off-topic neighbors. MMR still runs on the vector-only fallback path. +- **Canonical score**: every search path sets `SearchChunk.score`, and every downstream stage (sorting, filtering, greedy set cover, concept boost, reranker blending) compares only that field. `distance`, `bm25_score`, and the legacy `relevance_score` remain as provenance. +- **Abstention**: the canonical score is [0, 1] with fixed meaning (0.5 = top of one arm), so `min_relevance_score` is a usable floor: when every retrieved chunk falls below it, ask refuses instead of feeding noise as context. Raw RRF sums (~0.016-0.033 total range) made any threshold meaningless. - **max_distance** applies to rows whose *only* signal is a far vector match; a row the BM25 arm also matched keeps its standing regardless of distance, since dropping it would re-bury exactly the identifier hits fusion exists to preserve. - **When it helps**: queries with specific terms, function names, error messages, exact phrases, document identifiers. #### MMR Diversity -**Always on.** Maximal Marginal Relevance prevents near-duplicate chunks from filling all result slots. Runs on the fused candidate pool, with the canonical score as the relevance term, so diversity selection cannot re-demote a lexically-certain hit whose vector similarity happens to be weak. +**Vector-only path.** Maximal Marginal Relevance prevents near-duplicate chunks from filling all result slots when search runs without a text query (or hybrid falls back). It does not run on hybrid results, where the fused ordering is final. - **Paper**: Carbonell & Goldstein 1998, "[The Use of MMR, Diversity-Based Reranking](https://dl.acm.org/doi/10.1145/290941.291025)" - **Default**: λ=0.5 (equal weight to relevance and diversity). This is the standard default from the original paper. diff --git a/docs/usage.md b/docs/usage.md index dfea3b398..4e377e158 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -785,10 +785,8 @@ reason the defaults are the defaults. | `LILBEE_EXPANSION_SKIP_GAP` | `0.15` | Minimum relative raw-score gap between top-1 and top-2, `(top - second) / top`, for expansion to skip | | `LILBEE_EXPANSION_GUARDRAILS` | `true` | Filter expansion variants whose embedding drifts too far from the original query | | `LILBEE_EXPANSION_SIMILARITY_THRESHOLD` | `0.5` | Minimum query-variant cosine similarity to survive the guardrail | -| `LILBEE_CANDIDATE_MULTIPLIER` | `3` | Per-arm retrieval depth as a multiple of top_k (hybrid overfetch; also the vector-only MMR pool) | -| `LILBEE_FUSION_OVERFETCH_FLOOR` | `50` | Minimum rows fetched per retrieval arm before fusion, regardless of top_k | +| `LILBEE_CANDIDATE_MULTIPLIER` | `3` | Vector-only candidate pool as a multiple of top_k, feeding MMR reranking | | `LILBEE_ENTITY_EXTRACTION` | `false` | Extract typed entities at ingest for exact count answers (needs a reviewed `entity_schema.json`; see `lilbee entities`) | -| `LILBEE_FUSION_ALPHA` | `0.6` | Vector-arm weight in hybrid score fusion; the BM25 arm gets the complement | | `LILBEE_MIN_RELEVANCE_SCORE` | `0.0` | Abstention floor against the canonical [0, 1] relevance score; when every result falls below it, ask refuses instead of answering from noise | | `LILBEE_HISTORY_REWRITE` | `true` | Condense follow-up questions into standalone retrieval queries using chat history | | `LILBEE_INTENT_ROUTING` | `true` | Route document-name lookups to exact retrieval and count questions to a full-corpus scan | diff --git a/src/lilbee/app/settings_map.py b/src/lilbee/app/settings_map.py index 5edc22298..b2071ff8b 100644 --- a/src/lilbee/app/settings_map.py +++ b/src/lilbee/app/settings_map.py @@ -799,18 +799,6 @@ def get_default(key: str) -> object: group=SettingGroup.RETRIEVAL, help_text="Candidate-pool multiplier over top_k before reranking", ), - "fusion_alpha": SettingDef( - float, - nullable=False, - group=SettingGroup.RETRIEVAL, - help_text="Vector-arm weight in hybrid fusion (1.0 = pure vector, 0.0 = pure lexical)", - ), - "fusion_overfetch_floor": SettingDef( - int, - nullable=False, - group=SettingGroup.RETRIEVAL, - help_text="Minimum rows fetched per retrieval arm before fusion", - ), "history_rewrite": SettingDef( bool, nullable=False, diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index f24b8a554..4b6790d97 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -98,7 +98,7 @@ class Config(BaseSettings): # Abstention floor against the canonical [0, 1] relevance score # (0.0 = no filtering). When every retrieved chunk falls below it, ask # refuses instead of feeding noise as context; useful values start - # around 0.05-0.15 depending on fusion_alpha. + # around 0.02-0.05 against the fused reciprocal-rank score. min_relevance_score: float = ConfigField(default=0.0, ge=0.0, writable=True) adaptive_threshold: bool = Field(default=False) rag_system_prompt: str = ConfigField( @@ -188,20 +188,11 @@ class Config(BaseSettings): # (Carbonell & Goldstein 1998). mmr_lambda: float = ConfigField(default=0.5, ge=0.0, le=1.0, writable=True) - # Per-arm retrieval depth as a multiple of top_k: hybrid overfetches each - # arm this deep (floored at 50) before fusion, and the vector-only path - # retrieves this many extra candidates for MMR reranking. + # Vector-only search retrieves this many candidates per final result so + # MMR reranking has a pool to diversify from. Hybrid search ignores it: + # fusion arms stay exactly top_k deep. candidate_multiplier: int = ConfigField(default=3, ge=1, writable=True) - # Vector-arm weight in hybrid score fusion; the BM25 arm gets the - # complement. 1.0 = pure vector, 0.0 = pure lexical. - fusion_alpha: float = ConfigField(default=0.6, ge=0.0, le=1.0, writable=True) - - # Minimum rows fetched per arm before fusion, regardless of top_k and - # candidate_multiplier. Answer-level sweeps found deeper pools dilute the - # fused top-k, so this is tunable rather than fixed. - fusion_overfetch_floor: int = ConfigField(default=50, ge=1, writable=True) - # Chunk count at/above which sync builds an approximate (ANN) vector index # so search stays fast at millions of vectors. Below this, search uses exact # flat scan (faster and exact for small vaults). 0 disables the ANN index. diff --git a/src/lilbee/data/store/core.py b/src/lilbee/data/store/core.py index 7ac11f847..8bb5b42e1 100644 --- a/src/lilbee/data/store/core.py +++ b/src/lilbee/data/store/core.py @@ -606,31 +606,25 @@ def _hybrid_search( max_distance: float, chunk_type: ChunkType | None = None, ) -> list[SearchChunk]: - """Dual-arm retrieval fused by score, then MMR-selected down to top_k. - - Each arm is overfetched well past ``top_k`` so fusion sees rows ranked - just outside the final window in both arms; the previous rank-fused - path capped each arm at ``top_k``, hiding those rows entirely. MMR - runs on the canonical fused score, so lexical-only hits keep the - standing fusion gave them. + """Dual-arm retrieval fused by reciprocal rank; the fused ordering is final. + + Each arm fetches exactly ``top_k`` rows. Deeper pools measurably hurt + rank fusion: rows a single arm is certain about score a fixed 0.5, + while rows both arms rank mid-pool accumulate two contributions, so + widening the arms floods the fused top-k with both-arm mediocrity and + buries single-arm certainty (lexical identifier hits above all). + + No diversity selection runs here either. For lexical queries the + relevant passages are often mutually similar (they quote the same + identifiers), which is exactly what MMR penalizes: selecting the + fused pool through MMR measurably traded relevant lexical hits for + diverse off-topic neighbors on graded evaluation. """ - overfetch = max( - top_k * self._config.candidate_multiplier, self._config.fusion_overfetch_floor - ) fused = fuse_arms( - self._vector_arm(table, query_vector, overfetch, chunk_type), - self._fts_arm(table, query_text, overfetch, chunk_type), - self._config.fusion_alpha, + self._vector_arm(table, query_vector, top_k, chunk_type), + self._fts_arm(table, query_text, top_k, chunk_type), ) fused = _drop_unsupported_far_rows(fused, max_distance) - if len(fused) > top_k: - fused = mmr_rerank( - query_vector, - fused, - top_k, - self._config.mmr_lambda, - relevance_scores=[r.score or 0.0 for r in fused], - ) return fused[:top_k] def _filter_and_rerank( diff --git a/src/lilbee/data/store/fusion.py b/src/lilbee/data/store/fusion.py index 681fa2158..157784f47 100644 --- a/src/lilbee/data/store/fusion.py +++ b/src/lilbee/data/store/fusion.py @@ -1,24 +1,32 @@ -"""Score-aware fusion of the vector and BM25 retrieval arms. +"""Reciprocal-rank fusion of the vector and BM25 retrieval arms. -Rank-reciprocal fusion (the previous mechanism) discards score magnitude by -construction: it cannot tell an arm that was certain from an arm that was -guessing, so one strong lexical hit for an identifier query drowns under -mediocre dense neighbors. Fusing normalized scores keeps that information: +Each arm contributes ``(K + 1) / (K + rank)`` for the rows it retrieved +(rank is 1-based, K = 60, the standard RRF constant); a row's canonical +score is the mean of its two arm contributions, so it lives in [0, 1] and +a row ranked first by both arms scores exactly 1. Rows seen by only one +arm score half of that arm's contribution, which still places an arm's +top hit above every row deep in the other arm: the property that keeps a +lexical-only identifier match visible next to dense neighbors. - score = alpha * vector_similarity + (1 - alpha) * normalized_bm25 - -Vector similarity is ``1 - cosine_distance`` clamped to [0, 1], an absolute -signal. BM25 is unbounded and corpus-dependent, so it is normalized against -the top score of the result list (rank-stable, in (0, 1]). A row seen by only -one arm scores zero on the other, which is itself signal: lexical-only rows -survive fusion on their BM25 strength instead of being invisible to a -rank-based scheme fed from a shallow pool. +Rank fusion is deliberate. A convex combination of normalized raw scores +(``alpha * vector_similarity + (1 - alpha) * normalized_bm25``) was tried +here and measurably regressed graded precision: cosine similarities sit +in a high narrow band, giving every dense neighbor a floor that outranks +lexically-certain rows, and no blend weight fixes that asymmetry. Ranks +are scale-free, so neither arm's score distribution can crowd out the +other. Arm depth matters as much as the formula: rows both arms rank +mid-pool accumulate two contributions, so deep candidate pools crowd out +single-arm certainty; the hybrid path therefore feeds fusion arms of +exactly ``top_k`` rows. """ from __future__ import annotations from .types import SearchChunk +# Standard RRF smoothing constant (Cormack, Clarke & Buettcher 2009). +_RRF_K = 60 + def vector_similarity(distance: float) -> float: """Cosine distance to canonical [0, 1] similarity (distance spans [0, 2]).""" @@ -42,31 +50,28 @@ def _key(chunk: SearchChunk) -> tuple[str, int]: return (chunk.source, chunk.chunk_index) +def _rank_weight(rank: int) -> float: + """Reciprocal-rank contribution in (0, 1]; 1.0 at rank 1.""" + return (_RRF_K + 1) / (_RRF_K + rank) + + def fuse_arms( vector_rows: list[SearchChunk], fts_rows: list[SearchChunk], - alpha: float, ) -> list[SearchChunk]: - """Merge the two arms into one list scored by convex combination. + """Merge the two arms into one list scored by reciprocal rank. - ``alpha`` weights the vector arm; ``1 - alpha`` the BM25 arm. Rows found - by both arms carry both provenance fields. The result is sorted by - ``score`` descending and deduplicated on ``(source, chunk_index)``. + Both arms weigh equally. Rows found by both arms carry both provenance + fields (``distance`` from the vector arm, ``bm25_score`` from the FTS + arm). The result is sorted by ``score`` descending and deduplicated on + ``(source, chunk_index)``. """ - bm25_norms = dict( - zip( - (_key(r) for r in fts_rows), - normalized_bm25([r.bm25_score or 0.0 for r in fts_rows]), - strict=True, - ) - ) merged: dict[tuple[str, int], SearchChunk] = {} - for row in vector_rows: - sim = vector_similarity(row.distance) if row.distance is not None else 0.0 - merged[_key(row)] = row.model_copy(update={"score": alpha * sim}) - for row in fts_rows: + for rank, row in enumerate(vector_rows, start=1): + merged[_key(row)] = row.model_copy(update={"score": _rank_weight(rank) / 2}) + for rank, row in enumerate(fts_rows, start=1): key = _key(row) - lexical = (1.0 - alpha) * bm25_norms[key] + lexical = _rank_weight(rank) / 2 seen = merged.get(key) if seen is None: merged[key] = row.model_copy(update={"score": lexical}) diff --git a/src/lilbee/data/store/ranking.py b/src/lilbee/data/store/ranking.py index cb2c1e58f..5904269c1 100644 --- a/src/lilbee/data/store/ranking.py +++ b/src/lilbee/data/store/ranking.py @@ -25,7 +25,6 @@ def mmr_rerank( results: list[SearchChunk], top_k: int, mmr_lambda: float | None = None, - relevance_scores: list[float] | None = None, ) -> list[SearchChunk]: """Maximal Marginal Relevance: select diverse results. Algorithm: Carbonell & Goldstein 1998, @@ -36,12 +35,6 @@ def mmr_rerank( 0.0 = maximum diversity, 1.0 = pure relevance. Defaults to ``cfg.mmr_lambda`` (0.5). - ``relevance_scores`` overrides the relevance term (one value per result, - higher = better). Fused hybrid results pass their canonical score here so - a lexically-certain hit with weak vector similarity keeps its standing; - without the override, relevance is cosine similarity to the query, which - would silently re-demote exactly the rows fusion promoted. - Complexity: O(top_k · N · D) time, O(N · D) space for N candidates of dimension D. Each outer iteration updates a running max-redundancy vector via one matmul rather than recomputing pairs pairwise. @@ -64,10 +57,7 @@ def mmr_rerank( query_norm = float(np.linalg.norm(query)) or 1.0 query_unit = query / query_norm - if relevance_scores is not None: - relevance = np.asarray(relevance_scores, dtype=np.float32) - else: - relevance = cand_unit @ query_unit # shape (N,) + relevance = cand_unit @ query_unit # shape (N,) n = len(results) max_redundancy = np.zeros(n, dtype=np.float32) diff --git a/tests/test_store.py b/tests/test_store.py index ce4e0b67a..73514aa36 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -678,8 +678,9 @@ def test_lexical_only_match_survives_hybrid(self, store, test_config): results = store.search(query_vec, top_k=5, query_text="part number PX4471") assert any(r.source == "parts_catalog_214.pdf" for r in results) - def test_hybrid_overfetch_floor(self, store, test_config): - """Each arm is fetched at least fusion_overfetch_floor deep.""" + def test_hybrid_arms_fetch_exactly_top_k(self, store, test_config): + """Deeper pools flood rank fusion with both-arm mediocre rows, so + each arm is fetched exactly top_k deep.""" records = _make_records(n=3) store.add_chunks(records) store.ensure_fts_index() @@ -689,8 +690,8 @@ def test_hybrid_overfetch_floor(self, store, test_config): mock.patch.object(store, "_fts_arm", return_value=[]) as fts_arm, ): store.search(query_vec, top_k=3, query_text="chunk") - assert vec_arm.call_args[0][2] == test_config.fusion_overfetch_floor - assert fts_arm.call_args[0][2] == test_config.fusion_overfetch_floor + assert vec_arm.call_args[0][2] == 3 + assert fts_arm.call_args[0][2] == 3 def test_hybrid_vector_arm_applies_ann_recovery(self, store, test_config): """With a vector index present, the hybrid vector arm probes extra diff --git a/tests/test_store_fusion.py b/tests/test_store_fusion.py index f9993abbd..5e20e68e5 100644 --- a/tests/test_store_fusion.py +++ b/tests/test_store_fusion.py @@ -1,4 +1,4 @@ -"""Tests for score-aware fusion of the vector and BM25 arms.""" +"""Tests for reciprocal-rank fusion of the vector and BM25 arms.""" import pytest @@ -50,61 +50,128 @@ def test_both_arms_beat_either_alone(self): both = _chunk("a.md", 0, distance=0.3) vec_only = _chunk("b.md", 0, distance=0.3) lex = _chunk("a.md", 0, bm25=12.0) - fused = fuse_arms([both, vec_only], [lex], alpha=0.6) + fused = fuse_arms([both, vec_only], [lex]) scores = {(r.source, r.chunk_index): r.score for r in fused} assert scores[("a.md", 0)] > scores[("b.md", 0)] + def test_top_of_both_arms_scores_one(self): + fused = fuse_arms([_chunk("a.md", 0, distance=0.3)], [_chunk("a.md", 0, bm25=12.0)]) + assert fused[0].score == pytest.approx(1.0) + def test_lexical_only_row_survives_with_score(self): """The identifier case: a row invisible to the vector arm must carry - real fused weight from its BM25 strength alone.""" + real fused weight from its BM25-arm rank alone.""" vec = [_chunk("noise.md", i, distance=0.5) for i in range(3)] lex = [_chunk("catalog_482.pdf", 0, bm25=35.0)] - fused = fuse_arms(vec, lex, alpha=0.6) + fused = fuse_arms(vec, lex) lexical_row = next(r for r in fused if r.source == "catalog_482.pdf") - assert lexical_row.score == pytest.approx(0.4) + assert lexical_row.score == pytest.approx(0.5) - def test_certain_lexical_hit_outranks_mediocre_dense_neighbors(self): - """The pinpoint-document failure mode, inverted: top BM25 with weak - vector support must beat vector rows at middling similarity.""" - vec = [_chunk("noise.md", i, distance=0.8) for i in range(5)] + def test_top_lexical_hit_outranks_all_but_the_top_dense_neighbor(self): + """The pinpoint-document failure mode: an FTS-arm top hit unseen by + the vector arm must rank above every vector row except at most the + vector arm's own number one.""" + vec = [_chunk("noise.md", i, distance=0.1) for i in range(30)] lex = [_chunk("target.pdf", 0, bm25=30.0), _chunk("noise.md", 0, bm25=3.0)] - fused = fuse_arms(vec, lex, alpha=0.5) - assert fused[0].source == "target.pdf" + fused = fuse_arms(vec, lex) + rank = next(i for i, r in enumerate(fused) if r.source == "target.pdf") + assert rank <= 1 - def test_alpha_one_is_pure_vector(self): - fused = fuse_arms( - [_chunk("v.md", 0, distance=0.2)], [_chunk("l.md", 0, bm25=50.0)], alpha=1.0 - ) - scores = {r.source: r.score for r in fused} - assert scores["v.md"] == pytest.approx(0.8) - assert scores["l.md"] == 0.0 - - def test_alpha_zero_is_pure_lexical(self): - fused = fuse_arms( - [_chunk("v.md", 0, distance=0.2)], [_chunk("l.md", 0, bm25=50.0)], alpha=0.0 - ) - scores = {r.source: r.score for r in fused} - assert scores["l.md"] == pytest.approx(1.0) - assert scores["v.md"] == 0.0 + def test_rank_order_ignores_score_magnitude(self): + """A wildly stronger BM25 score at rank 2 stays rank 2: fusion is + scale-free by construction.""" + lex = [_chunk("first.md", 0, bm25=5.0), _chunk("second.md", 0, bm25=500.0)] + fused = fuse_arms([], lex) + assert [r.source for r in fused] == ["first.md", "second.md"] def test_dedup_keeps_both_provenance_fields(self): - fused = fuse_arms( - [_chunk("a.md", 0, distance=0.3)], [_chunk("a.md", 0, bm25=9.0)], alpha=0.5 - ) + fused = fuse_arms([_chunk("a.md", 0, distance=0.3)], [_chunk("a.md", 0, bm25=9.0)]) assert len(fused) == 1 assert fused[0].distance == pytest.approx(0.3) assert fused[0].bm25_score == pytest.approx(9.0) def test_sorted_descending_by_score(self): fused = fuse_arms( - [_chunk("far.md", 0, distance=1.5), _chunk("near.md", 0, distance=0.1)], + [_chunk("near.md", 0, distance=0.1), _chunk("far.md", 0, distance=1.5)], [], - alpha=1.0, ) assert [r.source for r in fused] == ["near.md", "far.md"] assert all(r.score is not None for r in fused) +class TestRegressionMechanism: + """Synthetic reproduction of the graded-A/B regression shape: a lexical + query whose relevant passages are mutually similar (they all quote the + same identifiers) competing against semantically-generic neighbors.""" + + @staticmethod + def _relevant(i, dim=8): + # Relevant rows: strong BM25, decent query-sim, and nearly identical + # to EACH OTHER (they all quote the same identifier table). + v = [0.9] * dim + v[0] += i * 0.001 + return SearchChunk( + source=f"ledger_{i}.txt", + content_type="text", + chunk_type="raw", + page_start=0, + page_end=0, + line_start=0, + line_end=0, + chunk=f"identifier table row {i}", + chunk_index=i, + vector=v, + distance=0.45, + bm25_score=30.0 - i, + ) + + @staticmethod + def _generic(i, dim=8): + # Generic rows: no lexical support, slightly better query-sim, + # mutually diverse. + v = [0.1] * dim + v[i % dim] = 0.9 + return SearchChunk( + source=f"essay_{i}.txt", + content_type="text", + chunk_type="raw", + page_start=0, + page_end=0, + line_start=0, + line_end=0, + chunk=f"general discussion {i}", + chunk_index=i, + vector=v, + distance=0.38, + bm25_score=None, + ) + + def test_fusion_alone_keeps_the_lexical_cluster(self): + """Rank fusion by itself keeps the both-arm relevant rows above + vector-only generics: the fusion ordering is not the regression.""" + relevant = [self._relevant(i) for i in range(15)] + generic = [self._generic(i) for i in range(35)] + fused = fuse_arms(relevant + generic, [r.model_copy() for r in relevant]) + top12 = fused[:12] + assert sum(1 for r in top12 if r.source.startswith("ledger")) >= 10 + + def test_mmr_diversifies_away_the_relevant_cluster(self): + """MMR at the default lambda over the fused pool demotes the mutually + similar relevant rows: the regression mechanism, and why the hybrid + path returns the fused ordering without diversity selection.""" + from lilbee.data.store import mmr_rerank + + relevant = [self._relevant(i) for i in range(15)] + generic = [self._generic(i) for i in range(35)] + fused = fuse_arms(relevant + generic, [r.model_copy() for r in relevant]) + query_vec = [0.5] * 8 + selected = mmr_rerank(query_vec, fused, 12, 0.5) + kept_relevant = sum(1 for r in selected if r.source.startswith("ledger")) + # The mutually-similar relevant set gets thinned hard in favor of + # diverse generics; this documents the mechanism the graded A/B saw. + assert kept_relevant < 10 + + class TestDropUnsupportedFarRows: def test_disabled_when_max_distance_zero(self): from lilbee.data.store.core import _drop_unsupported_far_rows From bcd13063068ce7769c2865ca153e04219412312a Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:09:23 -0400 Subject: [PATCH 60/77] docs: align remaining fusion wording with reciprocal rank Blast-radius sweep after the fusion reversal: the score-field comments, the pipeline flow diagram, the reranker section, and the tuning table still described convex score fusion and MMR on the hybrid path. Also corrects min_relevance_score guidance to the normalized rank scale (an arm's top hit scores 0.5, useful floors start near 0.4) and two drifted tuning-table rows (diversity default, expansion-skip semantics). --- docs/architecture.md | 13 ++++++------- docs/usage.md | 2 +- src/lilbee/core/config/model.py | 4 ++-- src/lilbee/data/store/core.py | 2 +- src/lilbee/data/store/types.py | 5 ++--- src/lilbee/server/models.py | 4 ++-- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index f47f920d8..777582351 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -559,9 +559,8 @@ flowchart TD GUARD --> MULTI[Multi-Query Search + Merge] MULTI --> DUAL - DUAL --> FUSE[Score Fusion → canonical score] - FUSE --> MMR[MMR Diversity on canonical score] - MMR --> CBOOST[Concept Boost] + DUAL --> FUSE[Reciprocal-Rank Fusion → canonical score] + FUSE --> CBOOST[Concept Boost] CBOOST --> GSORT[Global re-sort by score] GSORT --> ABST{All below min_relevance_score?} ABST -->|Yes| REFUSE[Grounded refusal] @@ -943,12 +942,12 @@ All settings are configurable via `LILBEE_*` environment variables, `config.toml | Setting | Default | Description | Caveats | |---------|---------|-------------|---------| | `LILBEE_MMR_LAMBDA` | `0.5` | Relevance vs diversity (0.0=diverse, 1.0=relevant) | 0.5 is the standard default from Carbonell & Goldstein 1998. Lower for broad exploratory queries. | -| `LILBEE_DIVERSITY_MAX_PER_SOURCE` | `3` | Max chunks returned per source document | Lower = more diverse sources. Higher = deeper coverage of a single document. | -| `LILBEE_CANDIDATE_MULTIPLIER` | `3` | How many extra candidates to retrieve for MMR | Higher = better diversity selection but slower. 3x is empirically effective. | +| `LILBEE_DIVERSITY_MAX_PER_SOURCE` | `5` | Max chunks returned per source document | Lower = more diverse sources. Higher = deeper coverage of a single document. | +| `LILBEE_CANDIDATE_MULTIPLIER` | `3` | Vector-only candidate pool multiplier feeding MMR | Higher = better diversity selection but slower; hybrid search ignores it. | | `LILBEE_QUERY_EXPANSION_COUNT` | `3` | Number of LLM-generated query variants | Each variant requires an embedding call. Set to 0 to disable expansion entirely for fastest search. | | `LILBEE_ADAPTIVE_THRESHOLD_STEP` | `0.2` | Distance filter widening increment | Only used when `LILBEE_ADAPTIVE_THRESHOLD=true`. Smaller = more granular adaptation but more filter iterations | -| `LILBEE_EXPANSION_SKIP_THRESHOLD` | `0.8` | BM25 score above which expansion is skipped | 90th percentile of sigmoid-normalized BM25 scores. Calibrate for your library. | -| `LILBEE_EXPANSION_SKIP_GAP` | `0.15` | Min score gap (top-1 minus top-2) to skip expansion | Approximately 1 std dev of typical score spread. Ensures the match isn't ambiguous. | +| `LILBEE_EXPANSION_SKIP_THRESHOLD` | `0.8` | BM25 confidence above which expansion is skipped | Confidence is s/(s+5) of the raw top BM25 score, unsaturated. 0 disables the skip. | +| `LILBEE_EXPANSION_SKIP_GAP` | `0.15` | Min relative gap between top-1 and top-2 raw BM25 to skip expansion | Fraction of the top score. Ensures the match isn't ambiguous. | | `LILBEE_EXPANSION_GUARDRAILS` | `true` | Validate expansion variants for drift | Prevents hallucinated variants at the cost of potentially filtering valid creative expansions | | `LILBEE_EXPANSION_SIMILARITY_THRESHOLD` | `0.5` | Minimum question↔variant cosine similarity for an expansion variant to survive the guardrail | Raise for stricter filtering, lower to keep more variants. Calibrate per embedding model. | | `LILBEE_MAX_CONTEXT_SOURCES` | `5` | Max chunks included in LLM context | More = more complete answers but higher latency and token cost | diff --git a/docs/usage.md b/docs/usage.md index 4e377e158..fc93bfeef 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1059,7 +1059,7 @@ export LILBEE_RERANKER_MODEL="bge-reranker-v2-m3" # any GGUF reranker export LILBEE_RERANK_CANDIDATES=20 # how many candidates to rerank ``` -Without a reranker set, hybrid search + MMR already provides good results for +Without a reranker set, fused hybrid search already provides good results for most use cases. Based on: Nogueira & Cho 2019 (Passage Re-ranking with BERT), Burges et al. diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index 4b6790d97..820770f8c 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -97,8 +97,8 @@ class Config(BaseSettings): max_distance: float = ConfigField(default=0.75, ge=0.0, writable=True) # Abstention floor against the canonical [0, 1] relevance score # (0.0 = no filtering). When every retrieved chunk falls below it, ask - # refuses instead of feeding noise as context; useful values start - # around 0.02-0.05 against the fused reciprocal-rank score. + # refuses instead of feeding noise as context. On the fused reciprocal-rank + # scale an arm's top hit scores 0.5, so useful floors start around 0.4. min_relevance_score: float = ConfigField(default=0.0, ge=0.0, writable=True) adaptive_threshold: bool = Field(default=False) rag_system_prompt: str = ConfigField( diff --git a/src/lilbee/data/store/core.py b/src/lilbee/data/store/core.py index 8bb5b42e1..50c7d8a5b 100644 --- a/src/lilbee/data/store/core.py +++ b/src/lilbee/data/store/core.py @@ -82,7 +82,7 @@ def _drop_unsupported_far_rows( A row the BM25 arm also matched keeps lexical support regardless of its vector distance; dropping it on distance alone would re-bury exactly the - identifier hits score fusion exists to preserve. + identifier hits rank fusion exists to preserve. """ if max_distance <= 0: return results diff --git a/src/lilbee/data/store/types.py b/src/lilbee/data/store/types.py index 14963f945..c1b4cef74 100644 --- a/src/lilbee/data/store/types.py +++ b/src/lilbee/data/store/types.py @@ -151,9 +151,8 @@ def _coerce_none_chunk_type(cls, v: str | None) -> str: bm25_score: float | None = Field(None, validation_alias="_score") rerank_score: float | None = None # Canonical relevance in [0, 1], set by the store on every search path: - # convex fusion of vector similarity and normalized BM25 on the hybrid - # path, clamped cosine similarity on vector-only, list-normalized BM25 on - # FTS-only probes. + # normalized reciprocal-rank fusion on the hybrid path, clamped cosine + # similarity on vector-only, list-normalized BM25 on FTS-only probes. score: float | None = None diff --git a/src/lilbee/server/models.py b/src/lilbee/server/models.py index 9ce7368ca..b53bb0ac7 100644 --- a/src/lilbee/server/models.py +++ b/src/lilbee/server/models.py @@ -123,8 +123,8 @@ class CleanedChunk(BaseModel): distance: float | None = None relevance_score: float | None = None rerank_score: float | None = None - # Canonical [0, 1] relevance from score fusion; the ranking signal HTTP - # clients should sort and threshold on (relevance_score is legacy RRF). + # Canonical [0, 1] relevance from retrieval fusion; the ranking signal + # HTTP clients should sort and threshold on (relevance_score is legacy). score: float | None = None page_start: int = 0 page_end: int = 0 From 06aabc1084f1fbeb799be31a4f6315efe5d1b447 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:42:22 -0400 Subject: [PATCH 61/77] fix(retrieval): retrieve rerank_candidates deep when a reranker is configured With a cross-encoder configured, the fused ordering is only candidate generation, but the ask path retrieved just top_k rows (12 by default, up to 2x with expansion), so the reranker's configured candidate count (60) never materialized and the cross-encoder could only reorder a dozen rows. build_rag_context now retrieves max(top_k, rerank_candidates) when reranker_model is set. The deep pool is safe there precisely because the reranker re-scores it; without a reranker the pool stays at top_k, where the fused order is final and deep rank-fused pools measurably bury single-arm certainty. --- src/lilbee/retrieval/query/searcher.py | 10 +++++- tests/test_query.py | 44 ++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index 398be7e24..43c0bc73e 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -681,7 +681,15 @@ def build_rag_context( # trims to the context window, keeping the document's head. results = known_item else: - results = self.search(retrieval_query, top_k=top_k, chunk_type=chunk_type) + retrieve_k = top_k or self._config.top_k + if self._config.reranker_model: + # A cross-encoder re-scores the pool, so fused order is only + # candidate generation here: retrieve deep enough that the + # reranker actually sees its configured candidate count. + # Without a reranker the fused order is final and stays at + # top_k, since deep rank-fused pools bury single-arm certainty. + retrieve_k = max(retrieve_k, self._config.rerank_candidates) + results = self.search(retrieval_query, top_k=retrieve_k, chunk_type=chunk_type) results = filter_results( results, self._config.max_distance, self._config.min_relevance_score ) diff --git a/tests/test_query.py b/tests/test_query.py index 731cad4e8..0b14e6dd0 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1994,6 +1994,50 @@ def test_reranker_called_when_configured(self, mock_svc): cfg.reranker_model = old +class TestRerankerPoolDepth: + def test_reranker_configured_retrieves_candidate_depth(self, mock_svc): + """With a cross-encoder re-scoring the pool, retrieval must fetch + rerank_candidates deep so the reranker sees its configured count.""" + mock_svc.store.search.return_value = [_make_result()] + mock_svc.reranker.rerank.return_value = [_make_result()] + old = cfg.reranker_model + cfg.reranker_model = "gpustack/bge-reranker-v2-m3-GGUF/bge-Q4_K_M.gguf" + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context("harbor light logistics") + finally: + cfg.reranker_model = old + cfg.query_expansion_count = 3 + assert mock_svc.store.search.call_args[1]["top_k"] == cfg.rerank_candidates + + def test_no_reranker_retrieves_exactly_top_k(self, mock_svc): + """Without a reranker the fused order is final; the pool stays at + top_k so deep rank fusion cannot bury single-arm certainty.""" + mock_svc.store.search.return_value = [_make_result()] + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context("harbor light logistics") + finally: + cfg.query_expansion_count = 3 + assert mock_svc.store.search.call_args[1]["top_k"] == cfg.top_k + + def test_explicit_top_k_above_candidates_wins(self, mock_svc): + """A caller asking for more rows than rerank_candidates keeps them.""" + mock_svc.store.search.return_value = [_make_result()] + mock_svc.reranker.rerank.return_value = [_make_result()] + old = cfg.reranker_model + cfg.reranker_model = "gpustack/bge-reranker-v2-m3-GGUF/bge-Q4_K_M.gguf" + cfg.query_expansion_count = 0 + try: + get_services().searcher.build_rag_context( + "harbor light logistics", top_k=cfg.rerank_candidates + 40 + ) + finally: + cfg.reranker_model = old + cfg.query_expansion_count = 3 + assert mock_svc.store.search.call_args[1]["top_k"] == cfg.rerank_candidates + 40 + + class TestAskStreamWithReranker: def test_reranker_called_when_configured(self, mock_svc): mock_svc.store.search.return_value = [_make_result()] From 9d4623d6eaf3a56fad4ad5801743f9052029b10c Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:17:26 -0400 Subject: [PATCH 62/77] fix(retrieval): resolve known-item queries on bare search, not just ask A query naming one document got routed to that document only through build_rag_context, so retrieval-only surfaces (HTTP /api/search, MCP search) returned similarity neighbors of the question's wording, which for 'summarize survey_482.pdf' is mostly noise. The differential probe exposed this: its known-item arm drives bare search and measured 0.000 on both sides because neither ever routed. Searcher.search now consults the known-item route (gated on intent_routing, skipped under explicit mode: prefixes, which force a strategy). A resolved document returns its head in document order at full confidence, capped at the standard search return budget. The ask path is unchanged; its earlier route check still short-circuits before search runs. --- docs/architecture.md | 2 +- src/lilbee/retrieval/query/searcher.py | 9 +++++ tests/test_query.py | 48 ++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index 777582351..802687010 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -582,7 +582,7 @@ flowchart TD - **Known-item**: a question naming a document (a filename, a quoted title, "document 482") resolves the reference against source metadata; a reference matching exactly one source returns that document's chunks in document order at full confidence. Similarity search retrieves neighbors of the question's *wording*, which for "summarize survey_482.pdf" is mostly noise. - **Aggregate**: "how many documents mention X" runs an exact scan over every chunk (streamed Arrow batches, flat memory) and answers with real numbers, no LLM involved. A count is a corpus property; the top 20 of half a million chunks structurally cannot count anything, and the model's only honest move was to hedge. Counts that need typed records the store does not hold are declined with a precise statement of what is countable. -- Every answering surface consults the same router (`Searcher.route_direct_answer`): CLI and TUI via ask, and each HTTP handler directly. +- Every answering surface consults the same router (`Searcher.route_direct_answer`): CLI and TUI via ask, and each HTTP handler directly. Known-item resolution also runs on bare retrieval (`Searcher.search`, so HTTP `/api/search` and MCP search), where a document-naming query returns that document's head in document order instead of similarity neighbors of its wording. #### History Condensation **On by default** (`LILBEE_HISTORY_REWRITE`). A follow-up question is condensed into a standalone retrieval query using the recent chat history (one small LLM call, skipped when there is no history). Retrieval sees only the query text: without this, "what about his brother?" is embedded and BM25-matched with its pronouns. The user's original wording still reaches the answering prompt. diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index 43c0bc73e..a096650a4 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -544,6 +544,15 @@ def search( if mode is not None: structured = self._search_structured(mode, clean_query, top_k, chunk_type=chunk_type) return self._apply_temporal_filter(structured, clean_query) + if self._config.intent_routing: + # A query naming one document wants that document on every + # retrieval surface, not just ask: without this, bare search + # (HTTP /api/search, MCP) returns similarity neighbors of the + # question's wording. The document's head, in document order, + # fills the standard return budget. + known_item = self._known_item_results(question) + if known_item: + return known_item[: top_k * 2] query_vec = self._embedder.embed_query(question) results = self._store.search( query_vec, diff --git a/tests/test_query.py b/tests/test_query.py index 0b14e6dd0..334a407b0 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1538,6 +1538,54 @@ class TestKnownItemRoute: def _source(self, filename): return {"filename": filename, "file_hash": "h", "ingested_at": "", "chunk_count": 2} + def test_bare_search_routes_named_document(self, mock_svc): + """The route covers every retrieval surface: bare search() (HTTP + /api/search, MCP) resolves a document-naming query to that document + instead of similarity neighbors of its wording.""" + mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] + mock_svc.store.get_chunks_by_source.return_value = [ + _make_result(source="survey_report.pdf", chunk="second", chunk_index=1), + _make_result(source="survey_report.pdf", chunk="first", chunk_index=0), + ] + results = get_services().searcher.search("summarize survey_report.pdf") + assert [r.chunk for r in results] == ["first", "second"] + assert all(r.score == 1.0 for r in results) + mock_svc.store.search.assert_not_called() + mock_svc.embedder.embed_query.assert_not_called() + + def test_bare_search_route_caps_at_return_budget(self, mock_svc): + """A huge named document fills the standard search return budget + (top_k*2) with its head, not the whole document.""" + mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] + mock_svc.store.get_chunks_by_source.return_value = [ + _make_result(source="survey_report.pdf", chunk=f"c{i}", chunk_index=i) + for i in range(100) + ] + results = get_services().searcher.search("summarize survey_report.pdf", top_k=5) + assert len(results) == 10 + assert [r.chunk_index for r in results] == list(range(10)) + + def test_bare_search_skips_route_when_intent_routing_off(self, mock_svc): + mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] + mock_svc.store.search.return_value = [_make_result()] + cfg.intent_routing = False + cfg.query_expansion_count = 0 + try: + get_services().searcher.search("summarize survey_report.pdf") + finally: + cfg.intent_routing = True + cfg.query_expansion_count = 3 + mock_svc.store.get_chunks_by_source.assert_not_called() + mock_svc.store.search.assert_called_once() + + def test_structured_mode_prefix_bypasses_route(self, mock_svc): + """An explicit mode: prefix is the user forcing a strategy; the + known-item route must not override it.""" + mock_svc.store.get_sources.return_value = [self._source("survey_report.pdf")] + mock_svc.store.bm25_probe.return_value = [] + get_services().searcher.search("term: survey_report.pdf") + mock_svc.store.get_chunks_by_source.assert_not_called() + def test_named_document_bypasses_similarity_search(self, mock_svc): """A question naming one resolvable document gets that document's chunks in document order, not a similarity ranking.""" From 3217a9c0d491beab14f256277711f1a8b5483f7a Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:40:37 -0400 Subject: [PATCH 63/77] fix(tests): stop a setup-wizard test starting a real model download test_enter_noop_outside_lilbee_app claimed to run without a TaskBarController, but its host class is a LilbeeApp subclass, so the controller was real, and with the installed-model scan patched to empty, Enter on a card submitted a genuine featured-model download. The worker thread outlived the test by a whole multi-gigabyte pull and starved whichever TUI test shared the xdist worker next, which is why a different test timed out each full-suite run while everything passed in isolation. The test now mocks start_download, asserts the submission it was silently making, and is renamed to say what it exercises. A new autouse conftest guard fails any test that leaves a live task-bar worker thread behind, so this class of leak fails at its owner instead of a random downstream victim. Two consecutive full-suite runs green after the fix; the flake previously hit three of four runs, and suite wall time dropped ~40% with no download saturating a worker. --- tests/conftest.py | 27 +++++++++++++++++++++++++++ tests/test_setup_task_routing.py | 13 ++++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1d73a8e3f..8e98fc5d9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -215,6 +215,33 @@ def overlay_reads_config_toml(monkeypatch): monkeypatch.delenv("LILBEE_SKIP_TOML_CONFIG", raising=False) +@pytest.fixture(autouse=True) +def _no_leaked_task_workers(): + """Fail the test that leaves a live task-bar worker thread behind. + + A leaked ``task-*`` worker (an unmocked model download, most expensively) + outlives its test by minutes and starves whichever TUI test shares the + xdist worker next, so the suite fails on a random victim instead of the + owner. A short grace join absorbs workers that are finishing a mocked + no-op target. + """ + before = {t.name for t in threading.enumerate() if t.name.startswith("task-")} + yield + leaked: list[str] = [] + for thread in threading.enumerate(): + if not thread.name.startswith("task-") or thread.name in before: + continue + thread.join(timeout=2.0) + if thread.is_alive(): + leaked.append(thread.name) + if leaked: + pytest.fail( + f"Test leaked live task-bar worker thread(s) {leaked}: the task " + "target (often a model download) must be mocked or joined before " + "the test returns, or it will starve later TUI tests on this worker." + ) + + @pytest.fixture(autouse=True) def _drain_textual_threads(): """Safety net: join non-daemon threads that outlive the test. diff --git a/tests/test_setup_task_routing.py b/tests/test_setup_task_routing.py index 4d52b4a73..f1d17408e 100644 --- a/tests/test_setup_task_routing.py +++ b/tests/test_setup_task_routing.py @@ -181,8 +181,11 @@ async def test_enter_does_not_resubmit_same_model_twice() -> None: @pytest.mark.asyncio -async def test_enter_noop_outside_lilbee_app() -> None: - """Without a TaskBarController, Enter on a card still records the selection.""" +async def test_enter_records_selection_and_submits_download() -> None: + """Enter on a not-installed card records the selection and submits its + download to the task bar. The submission is mocked: the real target + spawns a worker thread that outlives the test by a whole model download + and starves unrelated TUI tests on the same xdist worker.""" app = _PlainApp() with _patch_setup_scan(), _patch_setup_ram(): async with app.run_test(size=(120, 40)) as pilot: @@ -192,9 +195,13 @@ async def test_enter_noop_outside_lilbee_app() -> None: chat_cards = [c for c in wizard.query(ModelCard) if c.row.task == "chat"] first = chat_cards[0] mock_grid = GridSelect() - with patch("lilbee.app.settings.persistent_settings.update_values"): + with ( + patch("lilbee.app.settings.persistent_settings.update_values"), + patch.object(app.task_bar, "start_download") as start_download, + ): wizard._on_grid_selected(GridSelect.Selected(grid_select=mock_grid, widget=first)) assert first.selected is True + start_download.assert_called_once() @pytest.mark.asyncio From 8b19c55f33466435c08b49ff4fb486c87ce471fa Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:44:22 -0400 Subject: [PATCH 64/77] feat(retrieval): rerank_blend toggle for pure cross-encoder ordering The reranker always blended its scores with the retrieval fusion signal (position-aware, fusion-weighted up to 70% at the top), so measuring the reranker's own effect was impossible: a flat mean with heavy per-query reordering is indistinguishable between the reranker not separating relevant from plausible and the blend capping its upside. rerank_blend (default true) keeps the existing behavior; off applies the cross-encoder's ordering directly and stamps the normalized reranker score unblended. Also fixes a drifted default in the tuning docs (rerank candidates 60, not 20). --- docs/architecture.md | 3 ++- docs/usage.md | 1 + src/lilbee/app/settings_map.py | 6 ++++++ src/lilbee/core/config/model.py | 5 +++++ src/lilbee/retrieval/reranker.py | 18 ++++++++++++----- tests/test_reranker.py | 33 ++++++++++++++++++++++++++++++++ 6 files changed, 60 insertions(+), 6 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 802687010..2e2bbcfa8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -954,7 +954,8 @@ All settings are configurable via `LILBEE_*` environment variables, `config.toml | `LILBEE_HYDE` | `false` | Enable hypothetical document embeddings | Adds ~500ms per query. Best for vague queries where terminology doesn't match docs. | | `LILBEE_HYDE_WEIGHT` | `0.7` | Weight for HyDE results relative to original | Lower = less trust in hypothetical documents. 0.7 prevents fabricated vectors from dominating. | | `LILBEE_RERANKER_MODEL` | `""` | Cross-encoder model for reranking (empty = disabled) | Native GGUF (e.g. `bge-reranker-v2-m3`) or a remote name via the SDK backend. Only loaded when configured. | -| `LILBEE_RERANK_CANDIDATES` | `20` | Number of candidates to rerank | More = better precision but slower. 20 is a good balance. | +| `LILBEE_RERANK_CANDIDATES` | `60` | Number of candidates to rerank | More = deeper pool but slower. | +| `LILBEE_RERANK_BLEND` | `true` | Blend reranker scores with retrieval fusion, position-aware | Off = pure cross-encoder ordering, useful when measuring the reranker in isolation. | | `LILBEE_TEMPORAL_FILTERING` | `true` | Enable date-based result filtering | Only activates when temporal keywords are detected in the query | | `LILBEE_SHOW_REASONING` | `false` | Show reasoning model thinking process | For Qwen3/DeepSeek-R1 models that emit `` tags | | `LILBEE_CONCEPT_GRAPH` | `true` | Enable concept graph (LazyGraphRAG index) | Extracts noun phrases, builds co-occurrence graph, boosts search by concept overlap | diff --git a/docs/usage.md b/docs/usage.md index fc93bfeef..7f99a2446 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -708,6 +708,7 @@ something feels off. | `LILBEE_QUERY_EXPANSION_COUNT` | `3` | LLM-generated query variants per search. `0` disables expansion entirely for faster queries | | `LILBEE_RERANKER_MODEL` | *(none)* | GGUF cross-encoder reranker for a precision pass over top results. See [Cross-encoder reranking](#cross-encoder-reranking) | | `LILBEE_RERANK_CANDIDATES` | `60` | Candidates to rerank when a reranker is configured | +| `LILBEE_RERANK_BLEND` | `true` | Blend reranker scores with retrieval fusion; off = the reranker's own ordering stands | | `LILBEE_HYDE` | `false` | Enable Hypothetical Document Embeddings: an LLM drafts a hypothetical answer, that's embedded, and results are merged with the original query's. Adds ~500 ms per query; helps on vague questions | | `LILBEE_HYDE_WEIGHT` | `0.7` | How much to trust HyDE results relative to the direct query (0.0-1.0) | | `LILBEE_ADAPTIVE_THRESHOLD` | `false` | When too few results pass `LILBEE_MAX_DISTANCE`, widen the threshold step by step. Useful on small or noisy corpora | diff --git a/src/lilbee/app/settings_map.py b/src/lilbee/app/settings_map.py index b2071ff8b..a06f2af89 100644 --- a/src/lilbee/app/settings_map.py +++ b/src/lilbee/app/settings_map.py @@ -331,6 +331,12 @@ def get_default(key: str) -> object: group=SettingGroup.RETRIEVAL, help_text="Candidate pool size for reranking", ), + "rerank_blend": SettingDef( + bool, + nullable=False, + group=SettingGroup.RETRIEVAL, + help_text="Blend reranker scores with retrieval fusion (off = pure reranker order)", + ), "show_reasoning": SettingDef( bool, nullable=False, diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index 820770f8c..1fe8f85db 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -289,6 +289,11 @@ class Config(BaseSettings): # Candidate count sent to the reranker. rerank_candidates: int = ConfigField(default=60, ge=1, writable=True, public=True) + # Blend reranker scores with the retrieval fusion signal (position-aware). + # Off = the cross-encoder's own ordering stands unblended, which isolates + # the reranker's effect when measuring it. + rerank_blend: bool = ConfigField(default=True, writable=True, public=True) + # Date-range filter; only fires when a temporal keyword is detected. temporal_filtering: bool = ConfigField(default=True, writable=True) diff --git a/src/lilbee/retrieval/reranker.py b/src/lilbee/retrieval/reranker.py index a3f770c6a..9caccb6f8 100644 --- a/src/lilbee/retrieval/reranker.py +++ b/src/lilbee/retrieval/reranker.py @@ -104,11 +104,19 @@ def rerank( return results norm_scores = normalize_scores(scores) - fusion_norms = compute_fusion_norms(to_rerank) - blended = _blend_scores(to_rerank, norm_scores, fusion_norms) - blended_sorted = sorted(blended, key=lambda x: x.score, reverse=True) - - reranked = [chunk for _, chunk in blended_sorted] + if self._config.rerank_blend: + fusion_norms = compute_fusion_norms(to_rerank) + scored = _blend_scores(to_rerank, norm_scores, fusion_norms) + else: + # Pure cross-encoder ordering: no fusion blend, so the reranker's + # effect is unattenuated (and measurable in isolation). + scored = [ + ScoredChunk(s, c.model_copy(update={"rerank_score": s})) + for s, c in zip(norm_scores, to_rerank, strict=True) + ] + scored_sorted = sorted(scored, key=lambda x: x.score, reverse=True) + + reranked = [chunk for _, chunk in scored_sorted] return reranked + remainder diff --git a/tests/test_reranker.py b/tests/test_reranker.py index ec0a88531..1a2f0d15f 100644 --- a/tests/test_reranker.py +++ b/tests/test_reranker.py @@ -18,6 +18,7 @@ def _reset(): snapshot = { "reranker_model": cfg.reranker_model, "rerank_candidates": cfg.rerank_candidates, + "rerank_blend": cfg.rerank_blend, } yield for key, value in snapshot.items(): @@ -56,6 +57,38 @@ def _patch_provider(rerank_fn): return mock.patch("lilbee.app.services.get_services", return_value=services) +class TestPureOrderRerank: + def test_blend_off_applies_pure_cross_encoder_order(self, reranker): + """With rerank_blend off, ordering follows the reranker scores alone; + a strong fusion hit at the top no longer resists demotion.""" + cfg.reranker_model = _RERANKER_MODEL + cfg.rerank_blend = False + results = [ + _chunk("fusion_top.md", "strong hybrid hit", distance=0.05), + _chunk("mid.md", "middling", distance=0.5), + _chunk("reranker_pick.md", "cross-encoder favourite", distance=0.9), + ] + with _patch_provider(lambda q, docs: [0.1, 0.5, 0.9]): + reranked = reranker.rerank("query", results) + assert [r.source for r in reranked] == ["reranker_pick.md", "mid.md", "fusion_top.md"] + # rerank_score carries the normalized reranker score, unblended. + assert reranked[0].rerank_score == pytest.approx(1.0) + assert reranked[-1].rerank_score == pytest.approx(0.0) + + def test_blend_on_keeps_position_aware_default(self, reranker): + """Default behavior unchanged: the top fusion hit resists demotion.""" + cfg.reranker_model = _RERANKER_MODEL + cfg.rerank_blend = True + results = [ + _chunk("fusion_top.md", "strong hybrid hit", distance=0.05), + _chunk("mid.md", "middling", distance=0.5), + _chunk("reranker_pick.md", "cross-encoder favourite", distance=0.9), + ] + with _patch_provider(lambda q, docs: [0.1, 0.5, 0.9]): + reranked = reranker.rerank("query", results) + assert reranked[0].source == "fusion_top.md" + + class TestRerank: def test_returns_unchanged_when_no_model(self, reranker): cfg.reranker_model = "" From cb58b59c6055ea7761b1edf980afa299af6c61de Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:44:48 -0400 Subject: [PATCH 65/77] fix(tui): guard catalog refreshes against unmounted containers; wait out view transitions in tests A scheduled catalog refresh could fire while the screen was composing or being dismissed, query tab containers that don't exist, and crash the app with NoMatches; the catalog-quit test only ever passed by racing out before the refresh fired. Both refresh entrypoints now no-op when there is nothing mounted to paint into, placed before the row-cache update so a skipped refresh never marks itself done. The bracket-key navigation tests pressed through six screens on fixed pauses, but the view-switch guard deliberately drops a switch while the previous transition is in flight, so under a loaded xdist worker a keypress vanished and a random screen assertion failed. The cycling tests now wait for each transition, and the catalog-quit test waits on conditions instead of fixed pauses. --- src/lilbee/cli/tui/screens/catalog.py | 26 ++++++++++++ tests/test_tui.py | 58 ++++++++++++++++++++++++++- tests/test_tui_navigation.py | 25 ++++++++---- 3 files changed, 99 insertions(+), 10 deletions(-) diff --git a/src/lilbee/cli/tui/screens/catalog.py b/src/lilbee/cli/tui/screens/catalog.py index 449c1d328..69f6ed18c 100644 --- a/src/lilbee/cli/tui/screens/catalog.py +++ b/src/lilbee/cli/tui/screens/catalog.py @@ -12,6 +12,7 @@ from textual.app import ComposeResult from textual.binding import Binding, BindingType from textual.containers import Container, Horizontal, VerticalScroll +from textual.css.query import NoMatches from textual.events import Click, Key, MouseScrollDown from textual.message import Message from textual.screen import Screen @@ -338,6 +339,22 @@ def _list_for_tab(self, tab_id: str) -> ModelList: self._tab_list_cache[target] = widget return widget + def _grid_mounted(self) -> bool: + """Whether the active tab's grid container exists to paint into.""" + try: + self._grid_for_tab(self._active_tab_id_cache) + except NoMatches: + return False + return True + + def _list_mounted(self) -> bool: + """Whether the active tab's list widget exists to paint into.""" + try: + self._list_for_tab(self._active_tab_id_cache) + except NoMatches: + return False + return True + @property def _grid_container(self) -> VerticalScroll: return self._grid_for_tab(self._active_tab_id_cache) @@ -1128,7 +1145,14 @@ def _refresh_grid(self) -> None: update each existing ModelGrid via set_rows rather than tearing the container down and re-mounting from scratch. Avoids a 100% CPU spike on every "Browse more" return. + + A scheduled refresh can fire while the screen is composing or being + dismissed, when the tab containers aren't mounted; there is nothing + to paint then, and the guard runs before the row-cache update so the + next mounted refresh still repaints. """ + if not self._grid_mounted(): + return prep = self._prepare_grid_refresh() if prep is None: self._update_sort_label() @@ -1566,6 +1590,8 @@ def _on_model_list_selected(self, event: ModelList.Selected) -> None: def _refresh_list(self) -> None: """Rebuild the list view for the active tab; per-tab cache key skips no-op rebuilds.""" + if not self._list_mounted(): + return active_tab = self._active_tab_id_cache all_rows = self._sort_rows(self._build_rows()) if active_tab in TASK_TAB_IDS: diff --git a/tests/test_tui.py b/tests/test_tui.py index 0ea231536..3e46f8b44 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -361,6 +361,50 @@ async def test_empty_input_ignored(self, mock_catalog: mock.MagicMock) -> None: mock_send.assert_not_called() +class TestStatusStorageEntities: + def test_storage_panel_shows_entities_when_enabled(self): + from lilbee.app.status import EntityStatus + from lilbee.cli.tui.screens.status import _build_storage_content + from lilbee.core.config import cfg + + old_flag = cfg.entity_extraction + cfg.entity_extraction = True + try: + with mock.patch( + "lilbee.app.status.entity_status", + return_value=EntityStatus(types=["part_number"], rows=4), + ): + content = _build_storage_content(doc_count=1) + finally: + cfg.entity_extraction = old_flag + assert "4 extracted (part_number)" in content.plain + + def test_storage_panel_omits_entities_when_off(self): + from lilbee.cli.tui.screens.status import _build_storage_content + from lilbee.core.config import cfg + + old_flag = cfg.entity_extraction + cfg.entity_extraction = False + try: + content = _build_storage_content(doc_count=1) + finally: + cfg.entity_extraction = old_flag + assert "extracted" not in content.plain + + +class TestCatalogRefreshGuards: + def test_unmounted_refreshes_are_noops(self): + """A scheduled refresh landing while the screen is composing or being + dismissed has no containers to paint into and must not crash.""" + from lilbee.cli.tui.screens.catalog import CatalogScreen + + screen = CatalogScreen() + screen._refresh_grid() + screen._refresh_list() + assert not screen._grid_mounted() + assert not screen._list_mounted() + + class TestCatalogScreenAsync: @mock.patch("lilbee.cli.tui.screens.catalog.get_catalog") async def test_catalog_shows_featured(self, mock_catalog: mock.MagicMock) -> None: @@ -388,9 +432,19 @@ async def test_catalog_quit(self, mock_catalog: mock.MagicMock) -> None: await pilot.pause() catalog = CatalogScreen() app.push_screen(catalog) - await pilot.pause() + # Fixed single pauses race a loaded xdist worker: the key must not + # fire before the catalog is the active screen, and the dismissal + # needs time to process. Bounded condition loops on both sides. + for _ in range(40): + if app.screen is catalog: + break + await pilot.pause(0.05) + assert app.screen is catalog await pilot.press("q") - await pilot.pause() + for _ in range(40): + if not isinstance(app.screen, CatalogScreen): + break + await pilot.pause(0.05) # Catalog should be gone, chat screen visible assert not isinstance(app.screen, CatalogScreen) diff --git a/tests/test_tui_navigation.py b/tests/test_tui_navigation.py index c271bdd2d..c57b0b6a2 100644 --- a/tests/test_tui_navigation.py +++ b/tests/test_tui_navigation.py @@ -75,6 +75,20 @@ def _patch_chat_setup(): yield +async def _wait_for_screen(app, pilot, screen_type): + """Wait (bounded) for a view transition to land on *screen_type*. + + The view-switch guard deliberately drops a switch while the previous + transition is still in flight, so pressing again after one fixed pause + races a loaded worker; each press must wait for its transition. + """ + for _ in range(40): + if isinstance(app.screen, screen_type): + return + await pilot.pause(0.05) + raise AssertionError(f"Expected {screen_type.__name__}, got {type(app.screen).__name__}") + + async def test_bracket_keys_cycle_all_screens(): """Press ] through all 6 views from normal mode (Escape first on Chat).""" app = LilbeeApp() @@ -96,10 +110,7 @@ async def test_bracket_keys_cycle_all_screens(): ] for screen_type in expected: await pilot.press("right_square_bracket") - await pilot.pause() - assert isinstance(app.screen, screen_type), ( - f"Expected {screen_type.__name__}, got {type(app.screen).__name__}" - ) + await _wait_for_screen(app, pilot, screen_type) async def test_view_switch_guard_held_until_transition_completes(): @@ -162,12 +173,10 @@ async def test_bracket_keys_cycle_backward(): await pilot.pause() await pilot.press("left_square_bracket") - await pilot.pause() - assert isinstance(app.screen, FleetScreen) + await _wait_for_screen(app, pilot, FleetScreen) await pilot.press("left_square_bracket") - await pilot.pause() - assert isinstance(app.screen, TaskCenter) + await _wait_for_screen(app, pilot, TaskCenter) async def test_bracket_keys_work_from_settings(): From aa38e8a5d2abf5003c66f3365c929bca58e4881b Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:45:39 -0400 Subject: [PATCH 66/77] feat(entities): automatic sync-time lifecycle replaces the entities command Typed entity extraction was a three-step operator workflow: run induce, review a JSON schema by hand, run backfill. That contradicts how every other lilbee feature behaves, so turning entity_extraction on is now the whole interaction. Sync induces the schema from the indexed chunks on first run, extracts across the index, and picks up new files at ingest; editing entity_schema.json is detected by digest and re-applied on the next sync. Full passes always clear prior rows first, so interrupted passes and schema edits never double-count. No chat model at induction time defers with a log line and retries next sync. The entities command and its tests are gone; the same induction, extraction, and storage code now runs from the pipeline. Entity status (types, extracted rows) surfaces on all four fronts: CLI status, the TUI status screen's storage panel, GET /api/status (the response model previously would have silently dropped the section), and the MCP status tool. Schema induction is pinned to deterministic non-thinking generation via a new validated 'think' chat option: llama-server maps it to chat_template_kwargs.enable_thinking, hosted-API translators drop it. End-to-end on a real scanned manual, a small thinking model burned its entire token budget looping inside and emitted no JSON; think=false plus temperature 0 induced clean schemas on both test corpora (a PDF manual and twenty public-domain books, the latter extracting 337k entity rows across 7.5k chunks). --- docs/architecture.md | 29 +-- docs/usage.md | 5 +- src/lilbee/app/settings_map.py | 2 +- src/lilbee/app/status.py | 22 ++ src/lilbee/cli/commands/__init__.py | 2 - src/lilbee/cli/commands/entities.py | 160 ------------- src/lilbee/cli/helpers.py | 6 + src/lilbee/cli/tui/screens/status.py | 7 + src/lilbee/data/ingest/pipeline.py | 7 + src/lilbee/mcp_server.py | 9 + src/lilbee/providers/base.py | 5 + src/lilbee/providers/engine_params.py | 8 +- src/lilbee/providers/model_ref.py | 7 +- src/lilbee/retrieval/entities/__init__.py | 2 + src/lilbee/retrieval/entities/extractor.py | 11 +- src/lilbee/retrieval/entities/lifecycle.py | 173 ++++++++++++++ src/lilbee/server/models.py | 8 + tests/test_cli.py | 11 + tests/test_entities_cli.py | 224 ------------------ tests/test_entities_lifecycle.py | 249 +++++++++++++++++++++ tests/test_mcp.py | 13 ++ tests/test_model_ref.py | 28 +++ tests/test_server_handlers.py | 22 ++ 23 files changed, 601 insertions(+), 409 deletions(-) delete mode 100644 src/lilbee/cli/commands/entities.py create mode 100644 src/lilbee/retrieval/entities/lifecycle.py delete mode 100644 tests/test_entities_cli.py create mode 100644 tests/test_entities_lifecycle.py diff --git a/docs/architecture.md b/docs/architecture.md index 2e2bbcfa8..c8e6369b2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -703,22 +703,23 @@ answers from a scan instead of a hedge from top-k retrieval. Additive: the new table changes no existing schema, requires no migration, and a store without it behaves as if nothing was extracted. -Two phases, deliberately operator-driven: - -1. **Induction** (`lilbee entities induce`, cheap): an LLM reads a stratified - sample of the indexed chunks and proposes the corpus-specific type schema, - written to a reviewable `entity_schema.json`. A general NER tag set has no - notion of the identifier types a specific corpus carries; the schema is - the contract, and it is inspected (and edited) before anything expensive - runs. -2. **Application** (`lilbee entities backfill` for the existing index, plus - sync-time extraction for new files once the flag is on): each type is - found by the cheapest extractor that serves it: compiled regex for +Turning the setting on is the whole interaction; sync runs the lifecycle: + +1. **Induction** (first sync after enabling, cheap): an LLM reads a + stratified sample of the indexed chunks and proposes the corpus-specific + type schema, saved as `entity_schema.json` next to the index. A general + NER tag set has no notion of the identifier types a specific corpus + carries. The file is a tuning artifact, not a gate: edit a pattern and + the next sync detects the change (by digest) and re-extracts. +2. **Extraction** (same sync, then incrementally): each type is found by + the cheapest extractor that serves it: compiled regex for identifier-shaped types, spaCy labels for people/organizations/dates (when the `graph` extra's model is available), and an LLM only for types - neither can catch. Cost is dominated by how many LLM-kind types the - reviewed schema keeps. Backfill reads chunk text from the store, so no - documents are re-ingested and no embeddings are recomputed. + neither can catch. The full pass reads chunk text from the store (no + documents re-ingested, no embeddings recomputed) and always clears prior + rows first, so interrupted passes and schema edits never double-count. + New files extract at ingest. If no chat model is available for + induction, sync logs it and retries next time; nothing fails. Query-time effects: "how many distinct X" answers with exact distinct counts, and "how many X per Y" groups by chunk co-occurrence, both computed by full diff --git a/docs/usage.md b/docs/usage.md index 7f99a2446..d7e24201e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -508,9 +508,6 @@ lilbee ask "Explain this" --model qwen3 | `lilbee sync` | Re-index changed files | | `lilbee rebuild` | Nuke the database and re-ingest everything | | `lilbee export pages.parquet` | Write a per-page text dataset (parquet or jsonl, no vectors) | -| `lilbee entities induce` | Propose a typed entity schema from the indexed chunks (writes a reviewable `entity_schema.json`) | -| `lilbee entities backfill` | Extract typed entities from every stored chunk under the reviewed schema (no re-ingest; `--replace` clears prior rows first) | -| `lilbee entities status` | Show the entity schema, extracted row count, and whether sync-time extraction is on | | `lilbee import pages.parquet` | Import a dataset, re-embedding it with the current model | | `lilbee reset` | Factory reset. Deletes all documents and data | @@ -787,7 +784,7 @@ reason the defaults are the defaults. | `LILBEE_EXPANSION_GUARDRAILS` | `true` | Filter expansion variants whose embedding drifts too far from the original query | | `LILBEE_EXPANSION_SIMILARITY_THRESHOLD` | `0.5` | Minimum query-variant cosine similarity to survive the guardrail | | `LILBEE_CANDIDATE_MULTIPLIER` | `3` | Vector-only candidate pool as a multiple of top_k, feeding MMR reranking | -| `LILBEE_ENTITY_EXTRACTION` | `false` | Extract typed entities at ingest for exact count answers (needs a reviewed `entity_schema.json`; see `lilbee entities`) | +| `LILBEE_ENTITY_EXTRACTION` | `false` | Extract typed entities for exact count answers; fully automatic at sync (schema induced on first run, `entity_schema.json` editable to tune) | | `LILBEE_MIN_RELEVANCE_SCORE` | `0.0` | Abstention floor against the canonical [0, 1] relevance score; when every result falls below it, ask refuses instead of answering from noise | | `LILBEE_HISTORY_REWRITE` | `true` | Condense follow-up questions into standalone retrieval queries using chat history | | `LILBEE_INTENT_ROUTING` | `true` | Route document-name lookups to exact retrieval and count questions to a full-corpus scan | diff --git a/src/lilbee/app/settings_map.py b/src/lilbee/app/settings_map.py index a06f2af89..7cbe81275 100644 --- a/src/lilbee/app/settings_map.py +++ b/src/lilbee/app/settings_map.py @@ -147,7 +147,7 @@ def get_default(key: str) -> object: bool, nullable=False, group=SettingGroup.INGEST, - help_text="Extract typed entities at ingest (needs a reviewed entity_schema.json)", + help_text="Extract typed entities automatically at sync (schema induced on first run)", ), "semantic_chunking": SettingDef( bool, diff --git a/src/lilbee/app/status.py b/src/lilbee/app/status.py index cbcae885e..606f736a1 100644 --- a/src/lilbee/app/status.py +++ b/src/lilbee/app/status.py @@ -97,6 +97,13 @@ class SourceInfo(BaseModel): ingested_at: str +class EntityStatus(BaseModel): + """Entity-extraction section of a status response (present when enabled).""" + + types: list[str] + rows: int + + class StatusResult(BaseModel): """Full status response for the knowledge base.""" @@ -104,6 +111,7 @@ class StatusResult(BaseModel): config: StatusConfig sources: list[SourceInfo] total_chunks: int + entities: EntityStatus | None = None def gather_status() -> StatusResult: @@ -131,4 +139,18 @@ def gather_status() -> StatusResult: for s in sorted_sources ], total_chunks=total_chunks, + entities=entity_status(), ) + + +def entity_status() -> EntityStatus | None: + """Entity types + extracted row count; None while the feature is off.""" + if not cfg.entity_extraction: + return None + from lilbee.core.config import ENTITIES_TABLE + from lilbee.retrieval.entities import load_schema + + schema = load_schema(cfg.data_dir) + table = get_services().store.open_table(ENTITIES_TABLE) + rows = table.count_rows() if table is not None else 0 + return EntityStatus(types=[t.name for t in schema.types] if schema else [], rows=rows) diff --git a/src/lilbee/cli/commands/__init__.py b/src/lilbee/cli/commands/__init__.py index 68c906e74..df1c4c69c 100644 --- a/src/lilbee/cli/commands/__init__.py +++ b/src/lilbee/cli/commands/__init__.py @@ -12,7 +12,6 @@ from lilbee.cli.commands import ( agent_config, dataset, - entities, ingest_sync, memory, meta, @@ -32,7 +31,6 @@ app.command()(ingest_sync.add) app.command()(ingest_sync.chunks) app.command()(ingest_sync.remove) -app.command()(entities.entities) app.command(name="export")(dataset.export_cmd) app.command(name="import")(dataset.import_cmd) app.command()(search_chat.ask) diff --git a/src/lilbee/cli/commands/entities.py b/src/lilbee/cli/commands/entities.py deleted file mode 100644 index 39ad113cf..000000000 --- a/src/lilbee/cli/commands/entities.py +++ /dev/null @@ -1,160 +0,0 @@ -"""The ``lilbee entities`` command: schema induction, backfill, status. - -The mode is deliberately two-phase and operator-driven: ``induce`` samples -the indexed chunks and writes a reviewable ``entity_schema.json``; after the -schema is inspected (and edited if needed), ``backfill`` applies it across -every stored chunk without re-ingesting documents, and syncs keep new files -current once ``entity_extraction`` is enabled. -""" - -from __future__ import annotations - -from pathlib import Path - -import typer - -from lilbee.app.services import get_services -from lilbee.cli.app import apply_overrides, console, data_dir_option, global_option -from lilbee.cli.helpers import json_output -from lilbee.core.config import CHUNKS_TABLE, cfg - -# Chunks per extraction batch during backfill; bounds the working set. -_BACKFILL_BATCH = 2000 - - -def _sample_chunks(limit: int) -> list[str]: - """Stratified sample: chunks read evenly across the table so a corpus with - several document families contributes all of them.""" - store = get_services().store - table = store.open_table(CHUNKS_TABLE) - if table is None: - return [] - total = table.count_rows() - if total == 0: - return [] - step = max(1, total // limit) - texts: list[str] = [] - arrow = table.to_arrow().select(["chunk"]) - index = 0 - for batch in arrow.to_batches(max_chunksize=_BACKFILL_BATCH): - for text in batch.column("chunk").to_pylist(): - if index % step == 0 and len(texts) < limit: - texts.append(text) - index += 1 - return texts - - -def entities( - action: str = typer.Argument(help="induce | backfill | status"), - replace: bool = typer.Option( - False, - "--replace", - help="backfill only: clear previously extracted rows first, so repeated " - "backfills (after a pattern fix) don't double-count", - ), - data_dir: Path | None = data_dir_option, - use_global: bool = global_option, -) -> None: - """Typed entity extraction: propose a schema, backfill the index, or inspect state. - - ``induce`` writes entity_schema.json next to the index for review; - ``backfill`` extracts entities from every stored chunk under the reviewed - schema (no document re-ingest); ``status`` shows the schema and row counts. - """ - apply_overrides(data_dir=data_dir, use_global=use_global) - if action == "induce": - _induce() - elif action == "backfill": - _backfill(replace=replace) - elif action == "status": - _status() - else: - console.print(f"Unknown action {action!r}: expected induce, backfill, or status") - raise SystemExit(2) - - -def _induce() -> None: - from lilbee.retrieval.entities import induce_schema, save_schema - from lilbee.retrieval.entities.extractor import INDUCTION_SAMPLE_SIZE - - texts = _sample_chunks(INDUCTION_SAMPLE_SIZE) - if not texts: - console.print("Nothing indexed yet; sync documents before inducing a schema.") - raise SystemExit(1) - schema = induce_schema(texts, get_services().provider) - if schema is None: - console.print("Schema induction produced nothing usable; check the chat model and retry.") - raise SystemExit(1) - path = save_schema(schema, cfg.data_dir) - if cfg.json_mode: - json_output({"command": "entities", "action": "induce", "schema": str(path)}) - return - console.print(f"Proposed {len(schema.types)} types; review and edit {path}") - for entity_type in schema.types: - console.print(f" {entity_type.name} ({entity_type.kind.value})") - console.print("Then run: lilbee entities backfill") - - -def _backfill(*, replace: bool = False) -> None: - from lilbee.core.config import ENTITIES_TABLE - from lilbee.retrieval.concepts import concepts_available - from lilbee.retrieval.entities import ExtractorKind, extract_entities, load_schema - - schema = load_schema(cfg.data_dir) - if schema is None: - console.print("No reviewed entity_schema.json found; run: lilbee entities induce") - raise SystemExit(1) - store = get_services().store - table = store.open_table(CHUNKS_TABLE) - if table is None: - console.print("Nothing indexed yet; sync documents first.") - raise SystemExit(1) - if replace: - store.clear_table(ENTITIES_TABLE, "entity IS NOT NULL") - nlp = None - if any(t.kind is ExtractorKind.SPACY for t in schema.types) and concepts_available(): - from lilbee.retrieval.concepts.nlp import _ensure_spacy_model - - nlp = _ensure_spacy_model() - provider = None - if any(t.kind is ExtractorKind.LLM for t in schema.types): - provider = get_services().provider - written = 0 - columns = ["chunk", "source", "chunk_index", "page_start"] - arrow = table.to_arrow().select(columns) - for batch in arrow.to_batches(max_chunksize=_BACKFILL_BATCH): - records = batch.to_pylist() - rows = extract_entities(records, schema, provider=provider, nlp=nlp) - written += store.add_entities(rows) - if cfg.json_mode: - json_output({"command": "entities", "action": "backfill", "rows": written}) - return - console.print(f"Backfill complete: {written} entity rows extracted.") - - -def _status() -> None: - from lilbee.core.config import ENTITIES_TABLE - from lilbee.retrieval.entities import load_schema, schema_path - - schema = load_schema(cfg.data_dir) - table = get_services().store.open_table(ENTITIES_TABLE) - rows = table.count_rows() if table is not None else 0 - if cfg.json_mode: - json_output( - { - "command": "entities", - "action": "status", - "schema": str(schema_path(cfg.data_dir)) if schema else None, - "types": [t.name for t in schema.types] if schema else [], - "rows": rows, - "extraction_enabled": cfg.entity_extraction, - } - ) - return - if schema is None: - console.print("No entity schema; run: lilbee entities induce") - else: - names = ", ".join(t.name for t in schema.types) - console.print(f"Schema: {schema_path(cfg.data_dir)} ({names})") - console.print(f"Extracted rows: {rows}") - console.print(f"Sync-time extraction: {'on' if cfg.entity_extraction else 'off'}") diff --git a/src/lilbee/cli/helpers.py b/src/lilbee/cli/helpers.py index 051e54128..d747bdbc7 100644 --- a/src/lilbee/cli/helpers.py +++ b/src/lilbee/cli/helpers.py @@ -68,6 +68,12 @@ def render_status_result(status: StatusResult) -> Generator[RenderableType, None if status.config.enable_ocr is not None: ocr_label = "enabled" if status.config.enable_ocr else "disabled" yield f"[{theme.LABEL}]Vision OCR:[/{theme.LABEL}] {ocr_label}" + if status.entities is not None: + names = ", ".join(status.entities.types) or "schema pending (induced on next sync)" + yield ( + f"[{theme.LABEL}]Entities:[/{theme.LABEL}] " + f"{status.entities.rows} entities extracted ({names})" + ) yield "" if not status.sources: diff --git a/src/lilbee/cli/tui/screens/status.py b/src/lilbee/cli/tui/screens/status.py index 1bcb9250b..cfbcc640b 100644 --- a/src/lilbee/cli/tui/screens/status.py +++ b/src/lilbee/cli/tui/screens/status.py @@ -129,6 +129,13 @@ def _build_storage_content(doc_count: int) -> Content: _kv_line("Data dir", _collapse_home(cfg.data_dir)), _kv_line("Models dir", _collapse_home(cfg.models_dir)), ] + if cfg.entity_extraction: + from lilbee.app.status import entity_status + + section = entity_status() + if section is not None: + names = ", ".join(section.types) or "schema pending" + lines.append(_kv_line("Entities", f"{section.rows} extracted ({names})")) return Content("\n").join(lines) diff --git a/src/lilbee/data/ingest/pipeline.py b/src/lilbee/data/ingest/pipeline.py index 32675130d..4ab3d076c 100644 --- a/src/lilbee/data/ingest/pipeline.py +++ b/src/lilbee/data/ingest/pipeline.py @@ -533,6 +533,13 @@ async def sync( await incremental_update(set(added) | set(updated) | set(removed)) + # Entity lifecycle: induce-once, extract, and re-apply edited schemas. + # Runs every sync (cheap no-op when off or up to date) so flipping the + # setting on takes effect without any separate operation. + from lilbee.retrieval.entities.lifecycle import ensure_entities + + await to_ingest_thread(ensure_entities, cancel) + # Reconciliation guard against silent data loss: any on-disk document file that # ended up in neither the index nor the failed/skipped lists was dropped without # a signal. Surface it loudly instead of letting a whole dataset vanish quietly. diff --git a/src/lilbee/mcp_server.py b/src/lilbee/mcp_server.py index bc840db93..5ca9a13fb 100644 --- a/src/lilbee/mcp_server.py +++ b/src/lilbee/mcp_server.py @@ -226,9 +226,18 @@ def status() -> dict[str, Any]: for s in sorted(sources, key=lambda x: x["filename"]) ], "total_chunks": sum(s["chunk_count"] for s in sources), + "entities": _entity_status_dict(), } +def _entity_status_dict() -> dict[str, Any] | None: + """Entity types + extracted rows, mirroring the HTTP status section.""" + from lilbee.app.status import entity_status + + section = entity_status() + return section.model_dump() if section is not None else None + + @_tool async def sync(force_rebuild: bool = False, retry_skipped: bool = False) -> dict[str, Any]: """Sync the documents directory into the vector store. diff --git a/src/lilbee/providers/base.py b/src/lilbee/providers/base.py index dc07e9f30..78ca7c4f4 100644 --- a/src/lilbee/providers/base.py +++ b/src/lilbee/providers/base.py @@ -52,6 +52,11 @@ class LLMOptions(BaseModel): presence_penalty: float | None = None num_ctx: int | None = None stop: list[str] | None = None + # Thinking-template control for structured internal calls (schema + # induction and similar): a small thinking model can burn its whole + # token budget inside and emit nothing. llama-server maps this + # to chat_template_kwargs; hosted-API translators drop it. + think: bool | None = None def to_dict(self) -> dict[str, Any]: """Return only non-None values as a dict.""" diff --git a/src/lilbee/providers/engine_params.py b/src/lilbee/providers/engine_params.py index b9dd338b5..630d84705 100644 --- a/src/lilbee/providers/engine_params.py +++ b/src/lilbee/providers/engine_params.py @@ -82,8 +82,14 @@ def chat_options_to_kwargs(options: dict[str, Any] | None) -> dict[str, Any]: The output keys (``temperature``/``top_p``/``top_k``/``seed``/``max_tokens``/ ``repeat_penalty``) are accepted by llama-server's OpenAI body. ``top_k`` is kept (local llama.cpp honors it), unlike the SDK/API translator which drops it. + ``think`` becomes ``chat_template_kwargs.enable_thinking``, which thinking + templates honor and others ignore. """ - return normalize_generation_options(options) + kwargs = normalize_generation_options(options) + think = kwargs.pop("think", None) + if think is not None: + kwargs["chat_template_kwargs"] = {"enable_thinking": think} + return kwargs def resolve_model_path(model: str, registry: ModelRegistry | None = None) -> Path: diff --git a/src/lilbee/providers/model_ref.py b/src/lilbee/providers/model_ref.py index 42467731b..77bb4a1f9 100644 --- a/src/lilbee/providers/model_ref.py +++ b/src/lilbee/providers/model_ref.py @@ -170,7 +170,12 @@ def translate_options(options: dict[str, Any], ref: ProviderModelRef) -> dict[st it, so dropping it keeps the wire request clean. """ if not ref.is_api: - return filter_options(options) + filtered = filter_options(options) + # litellm has no chat_template_kwargs passthrough; thinking control + # only exists on the native llama-server path. + filtered.pop("think", None) + return filtered api_options = normalize_generation_options(options) api_options.pop("top_k", None) + api_options.pop("think", None) return api_options diff --git a/src/lilbee/retrieval/entities/__init__.py b/src/lilbee/retrieval/entities/__init__.py index a2ac56f59..8b733dba2 100644 --- a/src/lilbee/retrieval/entities/__init__.py +++ b/src/lilbee/retrieval/entities/__init__.py @@ -5,6 +5,7 @@ induce_schema, normalize_value, ) +from lilbee.retrieval.entities.lifecycle import ensure_entities from lilbee.retrieval.entities.schema import ( EntitySchema, EntityType, @@ -18,6 +19,7 @@ "EntitySchema", "EntityType", "ExtractorKind", + "ensure_entities", "extract_entities", "induce_schema", "load_schema", diff --git a/src/lilbee/retrieval/entities/extractor.py b/src/lilbee/retrieval/entities/extractor.py index e81a7ad1c..1a0ffcdf4 100644 --- a/src/lilbee/retrieval/entities/extractor.py +++ b/src/lilbee/retrieval/entities/extractor.py @@ -27,7 +27,10 @@ log = logging.getLogger(__name__) INDUCTION_SAMPLE_SIZE = 40 -INDUCTION_MAX_TOKENS = 1200 +# Thinking models spend most of their budget reasoning before the JSON +# appears (the reasoning is stripped afterward); 1200 starved them into +# emitting nothing parseable. +INDUCTION_MAX_TOKENS = 4096 # Chunks per LLM call in phase 2; larger batches save round-trips, smaller # ones keep each response comfortably parseable. LLM_EXTRACTION_BATCH = 8 @@ -99,7 +102,11 @@ def induce_schema(sample_texts: list[str], provider: LLMProvider) -> EntitySchem response = provider.chat( [{"role": "user", "content": prompt}], stream=False, - options={"num_predict": INDUCTION_MAX_TOKENS}, + # think=False: a small thinking model can loop inside + # until the budget is gone and emit no JSON at all. temperature 0: + # induction wants one deterministic, well-formed schema, not a + # creative sample that parses only some of the time. + options={"num_predict": INDUCTION_MAX_TOKENS, "think": False, "temperature": 0}, ) except Exception: log.warning("Entity schema induction failed at the provider", exc_info=True) diff --git a/src/lilbee/retrieval/entities/lifecycle.py b/src/lilbee/retrieval/entities/lifecycle.py new file mode 100644 index 000000000..4f88342c6 --- /dev/null +++ b/src/lilbee/retrieval/entities/lifecycle.py @@ -0,0 +1,173 @@ +"""Automatic entity-extraction lifecycle, run by sync. + +Turning ``entity_extraction`` on is the whole user interaction. Sync does +the rest: the first run samples the indexed chunks, induces a schema, and +extracts across the whole index; later runs extract new files at ingest +(see ``pipeline._build_entity_records``); and an edited +``entity_schema.json`` is detected by digest and re-applied across the +index on the next sync. The schema file is a manage-after-the-fact +artifact for tuning patterns, never a required step. + +Extraction is idempotent: every full pass clears previously extracted +rows first, so an interrupted pass or a schema edit can never +double-count. +""" + +from __future__ import annotations + +import hashlib +import logging +import threading +from typing import TYPE_CHECKING + +from lilbee.core.config import CHUNKS_TABLE, ENTITIES_TABLE, cfg +from lilbee.retrieval.entities.extractor import ( + INDUCTION_SAMPLE_SIZE, + extract_entities, + induce_schema, +) +from lilbee.retrieval.entities.schema import ( + EntitySchema, + ExtractorKind, + load_schema, + save_schema, + schema_path, +) + +if TYPE_CHECKING: + from lilbee.data.store import Store + +log = logging.getLogger(__name__) + +# Chunks per extraction batch during a full pass; bounds the working set. +_BACKFILL_BATCH = 2000 + +# Sidecar recording the digest of the last schema applied across the index, +# so an edited entity_schema.json triggers exactly one re-extraction. +_APPLIED_MARKER = "entity_schema.applied" + + +def _schema_digest() -> str | None: + path = schema_path(cfg.data_dir) + if not path.is_file(): + return None + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _applied_digest() -> str | None: + marker = cfg.data_dir / _APPLIED_MARKER + if not marker.is_file(): + return None + return marker.read_text().strip() or None + + +def _record_applied(digest: str | None) -> None: + if digest is None: + return + (cfg.data_dir / _APPLIED_MARKER).write_text(digest) + + +def ensure_entities(cancel: threading.Event | None = None) -> None: + """Bring extracted entities in line with the schema; no-op when off. + + Failure never fails the sync: a missing chat model defers induction to + the next sync with a log line, and a cancelled pass leaves the applied + marker unset so the next sync redoes the (idempotent) full pass. + """ + if not cfg.entity_extraction: + return + from lilbee.app.services import get_services + + store = get_services().store + schema = load_schema(cfg.data_dir) + if schema is None: + schema = _induce(store) + if schema is None: + return + elif _schema_digest() == _applied_digest(): + return # up to date; new files were covered at ingest + else: + log.info("Entity schema changed; re-extracting entities across the index") + if _full_pass(store, schema, cancel): + _record_applied(_schema_digest()) + + +def _induce(store: Store) -> EntitySchema | None: + """First-run schema induction from the indexed chunks. None defers.""" + from lilbee.app.services import get_services + + texts = _sample_chunks(store, INDUCTION_SAMPLE_SIZE) + if not texts: + log.info("Entity extraction is on but nothing is indexed yet; deferring") + return None + schema = induce_schema(texts, get_services().provider) + if schema is None: + log.warning("Entity schema induction produced nothing usable; retrying next sync") + return None + path = save_schema(schema, cfg.data_dir) + log.info( + "Induced entity schema (%d types) at %s; edit it to tune, the next sync re-applies", + len(schema.types), + path, + ) + return schema + + +def _sample_chunks(store: Store, limit: int) -> list[str]: + """Stratified sample: chunks read evenly across the table so a corpus + with several document families contributes all of them.""" + table = store.open_table(CHUNKS_TABLE) + if table is None: + return [] + total = table.count_rows() + if total == 0: + return [] + step = max(1, total // limit) + texts: list[str] = [] + arrow = table.to_arrow().select(["chunk"]) + index = 0 + for batch in arrow.to_batches(max_chunksize=_BACKFILL_BATCH): + for text in batch.column("chunk").to_pylist(): + if index % step == 0 and len(texts) < limit: + texts.append(text) + index += 1 + return texts + + +def _full_pass(store: Store, schema: EntitySchema, cancel: threading.Event | None) -> bool: + """Extract entities across every stored chunk. True when it completed. + + Clears previously extracted rows first so the pass is idempotent. + """ + table = store.open_table(CHUNKS_TABLE) + if table is None: + return False + store.clear_table(ENTITIES_TABLE, "entity IS NOT NULL") + nlp = None + if any(t.kind is ExtractorKind.SPACY for t in schema.types): + from lilbee.retrieval.concepts import concepts_available + + if concepts_available(): + from lilbee.retrieval.concepts.nlp import _ensure_spacy_model + + try: + nlp = _ensure_spacy_model() + except ImportError: + log.warning("spaCy model unavailable; skipping spaCy entity types") + provider = None + if any(t.kind is ExtractorKind.LLM for t in schema.types): + from lilbee.app.services import get_services + + provider = get_services().provider + written = 0 + columns = ["chunk", "source", "chunk_index", "page_start"] + arrow = table.to_arrow().select(columns) + for batch in arrow.to_batches(max_chunksize=_BACKFILL_BATCH): + if cancel is not None and cancel.is_set(): + log.info("Entity extraction cancelled; the next sync restarts the pass") + return False + records = batch.to_pylist() + rows = extract_entities(records, schema, provider=provider, nlp=nlp) + written += store.add_entities(rows) + log.info("Entity extraction complete: %d rows across the index", written) + return True diff --git a/src/lilbee/server/models.py b/src/lilbee/server/models.py index b53bb0ac7..708672fbf 100644 --- a/src/lilbee/server/models.py +++ b/src/lilbee/server/models.py @@ -163,6 +163,13 @@ class StatusConfigInfo(BaseModel): enable_ocr: bool | None = None +class StatusEntityInfo(BaseModel): + """Entity-extraction section of a status response (present when enabled).""" + + types: list[str] + rows: int + + class StatusResponse(BaseModel): """Response for GET /api/status.""" @@ -170,6 +177,7 @@ class StatusResponse(BaseModel): config: StatusConfigInfo sources: list[StatusSourceInfo] total_chunks: int + entities: StatusEntityInfo | None = None class HealthResponse(BaseModel): diff --git a/tests/test_cli.py b/tests/test_cli.py index 4b4d04344..cf26ed6b0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -104,6 +104,17 @@ def test_status_shows_models(self): assert "Chat model:" in result.output assert "Embeddings:" in result.output + def test_status_shows_entities_when_enabled(self): + cfg.entity_extraction = True + result = runner.invoke(app, ["status"]) + assert "entities extracted" in result.output + assert "schema pending" in result.output + + def test_status_hides_entities_when_off(self): + cfg.entity_extraction = False + result = runner.invoke(app, ["status"]) + assert "entities extracted" not in result.output + def test_status_shows_ocr_when_enabled(self): cfg.enable_ocr = True result = runner.invoke(app, ["status"]) diff --git a/tests/test_entities_cli.py b/tests/test_entities_cli.py deleted file mode 100644 index ef8a26807..000000000 --- a/tests/test_entities_cli.py +++ /dev/null @@ -1,224 +0,0 @@ -"""End-to-end tests for the ``lilbee entities`` command.""" - -import pytest -from typer.testing import CliRunner - -import lilbee.app.services as svc_mod -import lilbee.cli.commands # noqa: F401 (registers commands on the app) -from lilbee.cli.app import app -from lilbee.core.config import cfg -from lilbee.data.store import Store -from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind, save_schema - -runner = CliRunner() - - -@pytest.fixture() -def isolated(tmp_path, monkeypatch): - """A real Store on a tmp data root, pinned via --data-dir on every invoke. - - The CLI callback re-resolves the data root, so tests pass the flag - explicitly instead of trusting pre-set singleton paths. - """ - from tests.conftest import make_mock_services - - monkeypatch.delenv("LILBEE_DATA", raising=False) - snapshot = cfg.model_copy() - cfg.data_root = tmp_path - cfg.documents_dir = tmp_path / "documents" - cfg.data_dir = tmp_path / "data" - cfg.lancedb_dir = tmp_path / "data" / "lancedb" - cfg.data_dir.mkdir(parents=True, exist_ok=True) - store = Store(cfg) - services = make_mock_services(store=store) - svc_mod.set_services(services) - yield store - svc_mod.set_services(None) - for name in type(cfg).model_fields: - setattr(cfg, name, getattr(snapshot, name)) - - -def _invoke(action, tmp_root): - return runner.invoke(app, ["entities", action, "--data-dir", str(tmp_root)]) - - -def _index_chunks(store, texts): - dim = cfg.embedding_dim - store.add_chunks( - [ - { - "source": "catalog.txt", - "content_type": "text", - "chunk_type": "raw", - "page_start": 1, - "page_end": 1, - "line_start": 0, - "line_end": 0, - "chunk": text, - "chunk_index": i, - "vector": [0.1] * dim, - } - for i, text in enumerate(texts) - ] - ) - - -def _invoke_json(action, tmp_root): - return runner.invoke(app, ["--json", "entities", action, "--data-dir", str(tmp_root)]) - - -class TestEntitiesCommand: - def test_json_status_and_backfill_and_induce(self, isolated): - import json as jsonlib - from types import SimpleNamespace - - _index_chunks(isolated, ["part PX4471"]) - services = svc_mod.get_services() - services.provider.chat.return_value = SimpleNamespace( - text='{"types": [{"name": "part_number", "kind": "regex", ' - '"pattern": "PX\\\\d{4}", "description": "ids", "synonyms": []}]}' - ) - induced = _invoke_json("induce", cfg.data_root) - assert induced.exit_code == 0 - assert jsonlib.loads(induced.output)["action"] == "induce" - backfilled = _invoke_json("backfill", cfg.data_root) - assert jsonlib.loads(backfilled.output)["rows"] == 1 - status = _invoke_json("status", cfg.data_root) - payload = jsonlib.loads(status.output) - assert payload["rows"] == 1 - assert payload["types"] == ["part_number"] - - def test_backfill_with_spacy_and_llm_types(self, isolated): - from types import SimpleNamespace - from unittest import mock as umock - - _index_chunks(isolated, ["part PX4471 shipped"]) - save_schema( - EntitySchema( - types=[ - EntityType(name="person", kind=ExtractorKind.SPACY, pattern="PERSON"), - EntityType(name="vessel", kind=ExtractorKind.LLM, description="ships"), - EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}"), - ] - ), - cfg.data_dir, - ) - services = svc_mod.get_services() - services.provider.chat.return_value = SimpleNamespace(text="{}") - fake_nlp = umock.MagicMock(return_value=umock.MagicMock(ents=[])) - with ( - umock.patch("lilbee.retrieval.concepts.concepts_available", return_value=True), - umock.patch("lilbee.retrieval.concepts.nlp._ensure_spacy_model", return_value=fake_nlp), - ): - result = _invoke("backfill", cfg.data_root) - assert result.exit_code == 0 - assert "1 entity rows" in result.output - - def test_backfill_replace_clears_previous_rows(self, isolated): - """A repeated backfill after a pattern fix must not double-count.""" - _index_chunks(isolated, ["part PX4471"]) - save_schema( - EntitySchema( - types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] - ), - cfg.data_dir, - ) - _invoke("backfill", cfg.data_root) - _invoke("backfill", cfg.data_root) - assert isolated.entity_value_counts("part_number")[0] == 2 # appended: doubled - result = runner.invoke( - app, ["entities", "backfill", "--replace", "--data-dir", str(cfg.data_root)] - ) - assert result.exit_code == 0 - assert isolated.entity_value_counts("part_number") == (1, 1) - - def test_backfill_with_nothing_indexed(self, isolated): - save_schema( - EntitySchema( - types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] - ), - cfg.data_dir, - ) - result = _invoke("backfill", cfg.data_root) - assert result.exit_code == 1 - assert "sync documents" in result.output - - def test_induce_with_emptied_table(self, isolated): - """A chunks table whose rows were all removed samples nothing.""" - _index_chunks(isolated, ["part PX4471"]) - isolated.upsert_source("catalog.txt", "h", chunk_count=1) - isolated.remove_documents(["catalog.txt"]) - result = _invoke("induce", cfg.data_root) - assert result.exit_code == 1 - assert "sync documents" in result.output - - def test_status_before_anything(self, isolated): - result = _invoke("status", cfg.data_root) - assert result.exit_code == 0 - assert "No entity schema" in result.output - - def test_backfill_requires_schema(self, isolated): - result = _invoke("backfill", cfg.data_root) - assert result.exit_code == 1 - assert "induce" in result.output - - def test_backfill_extracts_over_stored_chunks(self, isolated): - """The no-re-ingest path: rows come from the chunks table alone.""" - _index_chunks(isolated, ["part PX4471 shipped", "parts PX9001 and PX4471"]) - save_schema( - EntitySchema( - types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] - ), - cfg.data_dir, - ) - result = _invoke("backfill", cfg.data_root) - assert result.exit_code == 0 - assert "3 entity rows" in result.output - mentions, distinct = isolated.entity_value_counts("part_number") - assert (mentions, distinct) == (3, 2) - - def test_status_after_backfill(self, isolated): - _index_chunks(isolated, ["part PX4471"]) - save_schema( - EntitySchema( - types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] - ), - cfg.data_dir, - ) - _invoke("backfill", cfg.data_root) - result = _invoke("status", cfg.data_root) - assert result.exit_code == 0 - assert "part_number" in result.output - assert "Extracted rows: 1" in result.output - - def test_induce_writes_reviewable_schema(self, isolated, monkeypatch): - from types import SimpleNamespace - - _index_chunks(isolated, ["part PX4471 shipped from the depot"]) - services = svc_mod.get_services() - services.provider.chat.return_value = SimpleNamespace( - text='{"types": [{"name": "part_number", "kind": "regex", ' - '"pattern": "PX\\\\d{4}", "description": "ids", "synonyms": []}]}' - ) - result = _invoke("induce", cfg.data_root) - assert result.exit_code == 0 - assert "review and edit" in result.output - assert (cfg.data_dir / "entity_schema.json").is_file() - - def test_induce_with_unusable_model_output(self, isolated): - from types import SimpleNamespace - - _index_chunks(isolated, ["part PX4471"]) - svc_mod.get_services().provider.chat.return_value = SimpleNamespace(text="no json") - result = _invoke("induce", cfg.data_root) - assert result.exit_code == 1 - assert "nothing usable" in result.output - - def test_unknown_action(self, isolated): - result = _invoke("frobnicate", cfg.data_root) - assert result.exit_code == 2 - - def test_induce_with_empty_index(self, isolated): - result = _invoke("induce", cfg.data_root) - assert result.exit_code == 1 - assert "sync documents" in result.output diff --git a/tests/test_entities_lifecycle.py b/tests/test_entities_lifecycle.py new file mode 100644 index 000000000..95bcc6a75 --- /dev/null +++ b/tests/test_entities_lifecycle.py @@ -0,0 +1,249 @@ +"""Tests for the automatic entity lifecycle run by sync.""" + +import threading +from types import SimpleNamespace +from unittest import mock + +import pytest + +import lilbee.app.services as svc_mod +from lilbee.core.config import cfg +from lilbee.data.store import Store +from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind, save_schema +from lilbee.retrieval.entities.lifecycle import _APPLIED_MARKER, ensure_entities + + +@pytest.fixture() +def isolated(tmp_path): + """A real Store on a tmp data root with entity extraction enabled.""" + from tests.conftest import make_mock_services + + snapshot = cfg.model_copy() + cfg.data_root = tmp_path + cfg.documents_dir = tmp_path / "documents" + cfg.data_dir = tmp_path / "data" + cfg.lancedb_dir = tmp_path / "data" / "lancedb" + cfg.data_dir.mkdir(parents=True, exist_ok=True) + cfg.entity_extraction = True + store = Store(cfg) + services = make_mock_services(store=store) + svc_mod.set_services(services) + yield store, services + svc_mod.set_services(None) + for name in type(cfg).model_fields: + setattr(cfg, name, getattr(snapshot, name)) + + +def _index_chunks(store, texts): + dim = cfg.embedding_dim + store.add_chunks( + [ + { + "source": "catalog.txt", + "content_type": "text", + "chunk_type": "raw", + "page_start": 1, + "page_end": 1, + "line_start": 0, + "line_end": 0, + "chunk": text, + "chunk_index": i, + "vector": [0.1] * dim, + } + for i, text in enumerate(texts) + ] + ) + + +def _part_schema(): + return EntitySchema( + types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] + ) + + +_INDUCED_JSON = ( + '{"types": [{"name": "part_number", "kind": "regex", ' + '"pattern": "PX\\\\d{4}", "description": "ids", "synonyms": []}]}' +) + + +class TestEnsureEntities: + def test_off_is_a_noop(self, isolated): + _store, services = isolated + cfg.entity_extraction = False + ensure_entities() + services.provider.chat.assert_not_called() + + def test_first_run_induces_and_extracts(self, isolated): + """Enabling the setting is the whole interaction: one sync pass + induces the schema, saves it, and extracts across the index.""" + store, services = isolated + _index_chunks(store, ["part PX4471 shipped", "parts PX9001 and PX4471"]) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + assert (cfg.data_dir / "entity_schema.json").is_file() + assert (cfg.data_dir / _APPLIED_MARKER).is_file() + mentions, distinct = store.entity_value_counts("part_number") + assert (mentions, distinct) == (3, 2) + + def test_nothing_indexed_defers_induction(self, isolated): + _store, services = isolated + ensure_entities() + services.provider.chat.assert_not_called() + assert not (cfg.data_dir / "entity_schema.json").is_file() + + def test_unusable_induction_retries_next_sync(self, isolated): + store, services = isolated + _index_chunks(store, ["part PX4471"]) + services.provider.chat.return_value = SimpleNamespace(text="no json") + ensure_entities() + assert not (cfg.data_dir / "entity_schema.json").is_file() + # Next sync with a working model succeeds. + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + assert store.entity_value_counts("part_number") == (1, 1) + + def test_up_to_date_schema_is_a_noop(self, isolated): + store, services = isolated + _index_chunks(store, ["part PX4471"]) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + with mock.patch("lilbee.retrieval.entities.lifecycle._full_pass") as full_pass: + ensure_entities() + full_pass.assert_not_called() + + def test_edited_schema_reextracts_without_double_counting(self, isolated): + """The manage-after-the-fact path: edit entity_schema.json, the next + sync detects the digest change and re-runs a replace-semantics pass.""" + store, _services = isolated + _index_chunks(store, ["part PX4471 near dock D-77"]) + save_schema(_part_schema(), cfg.data_dir) + ensure_entities() + assert store.entity_value_counts("part_number") == (1, 1) + # Edit: add a second type. Digest changes; next sync re-applies. + save_schema( + EntitySchema( + types=[ + EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}"), + EntityType(name="dock", kind=ExtractorKind.REGEX, pattern=r"D-\d{2}"), + ] + ), + cfg.data_dir, + ) + ensure_entities() + assert store.entity_value_counts("part_number") == (1, 1) # not doubled + assert store.entity_value_counts("dock") == (1, 1) + + def test_cancelled_pass_restarts_next_sync(self, isolated): + store, _services = isolated + _index_chunks(store, ["part PX4471"]) + save_schema(_part_schema(), cfg.data_dir) + cancelled = threading.Event() + cancelled.set() + ensure_entities(cancel=cancelled) + assert not (cfg.data_dir / _APPLIED_MARKER).is_file() + ensure_entities() + assert (cfg.data_dir / _APPLIED_MARKER).is_file() + assert store.entity_value_counts("part_number") == (1, 1) + + def test_spacy_and_llm_kinds_wire_their_tools(self, isolated): + store, services = isolated + _index_chunks(store, ["part PX4471 shipped"]) + save_schema( + EntitySchema( + types=[ + EntityType(name="person", kind=ExtractorKind.SPACY, pattern="PERSON"), + EntityType(name="vessel", kind=ExtractorKind.LLM, description="ships"), + EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}"), + ] + ), + cfg.data_dir, + ) + services.provider.chat.return_value = SimpleNamespace(text="{}") + fake_nlp = mock.MagicMock(return_value=mock.MagicMock(ents=[])) + with ( + mock.patch("lilbee.retrieval.concepts.concepts_available", return_value=True), + mock.patch("lilbee.retrieval.concepts.nlp._ensure_spacy_model", return_value=fake_nlp), + ): + ensure_entities() + fake_nlp.assert_called() + services.provider.chat.assert_called() + assert store.entity_value_counts("part_number") == (1, 1) + + def test_spacy_model_import_error_degrades(self, isolated, caplog): + store, _services = isolated + _index_chunks(store, ["part PX4471"]) + save_schema( + EntitySchema( + types=[ + EntityType(name="person", kind=ExtractorKind.SPACY, pattern="PERSON"), + EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}"), + ] + ), + cfg.data_dir, + ) + with ( + mock.patch("lilbee.retrieval.concepts.concepts_available", return_value=True), + mock.patch( + "lilbee.retrieval.concepts.nlp._ensure_spacy_model", + side_effect=ImportError("no model"), + ), + caplog.at_level("WARNING"), + ): + ensure_entities() + assert store.entity_value_counts("part_number") == (1, 1) # regex still ran + assert any("spaCy model unavailable" in r.message for r in caplog.records) + + def test_empty_chunks_table_skips_pass(self, isolated): + _store, _services = isolated + save_schema(_part_schema(), cfg.data_dir) + ensure_entities() + assert not (cfg.data_dir / _APPLIED_MARKER).is_file() + + def test_emptied_chunks_table_defers_induction(self, isolated): + """A chunks table whose rows were all removed samples nothing.""" + store, services = isolated + _index_chunks(store, ["part PX4471"]) + store.upsert_source("catalog.txt", "h", chunk_count=1) + store.remove_documents(["catalog.txt"]) + ensure_entities() + services.provider.chat.assert_not_called() + + def test_blank_applied_marker_reads_as_unapplied(self, isolated): + """A truncated marker file must trigger a (safe, idempotent) re-pass.""" + store, _services = isolated + _index_chunks(store, ["part PX4471"]) + save_schema(_part_schema(), cfg.data_dir) + (cfg.data_dir / _APPLIED_MARKER).write_text("") + ensure_entities() + assert store.entity_value_counts("part_number") == (1, 1) + assert (cfg.data_dir / _APPLIED_MARKER).read_text().strip() + + def test_missing_schema_digest_never_records_marker(self, isolated): + """_record_applied(None) is the no-op guard for a vanished schema file.""" + from lilbee.retrieval.entities.lifecycle import _record_applied, _schema_digest + + assert _schema_digest() is None + _record_applied(None) + assert not (cfg.data_dir / _APPLIED_MARKER).is_file() + + +class TestStatusEntities: + def test_status_reports_types_and_rows(self, isolated): + from lilbee.app.status import gather_status + + store, _services = isolated + _index_chunks(store, ["part PX4471"]) + save_schema(_part_schema(), cfg.data_dir) + ensure_entities() + result = gather_status() + assert result.entities is not None + assert result.entities.types == ["part_number"] + assert result.entities.rows == 1 + + def test_status_omits_section_when_off(self, isolated): + from lilbee.app.status import gather_status + + cfg.entity_extraction = False + result = gather_status() + assert result.entities is None diff --git a/tests/test_mcp.py b/tests/test_mcp.py index afacddd85..1b33d0629 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -257,6 +257,19 @@ def test_status_enable_ocr_none_by_default(self): result = status() assert result["config"]["enable_ocr"] is None + def test_status_includes_entities_when_enabled(self, mock_svc): + cfg.entity_extraction = True + mock_svc.store.open_table.return_value = None + try: + result = status() + finally: + cfg.entity_extraction = False + assert result["entities"] == {"types": [], "rows": 0} + + def test_status_omits_entities_when_off(self, mock_svc): + cfg.entity_extraction = False + assert status()["entities"] is None + def test_status_exposes_all_four_model_roles(self): """MCP status must expose vision + reranker slots so plugin clients see them.""" result = status() diff --git a/tests/test_model_ref.py b/tests/test_model_ref.py index e508f718b..cb0aecfeb 100644 --- a/tests/test_model_ref.py +++ b/tests/test_model_ref.py @@ -320,6 +320,34 @@ def test_empty_options(self) -> None: result = translate_options({}, ref) assert result == {} + def test_api_model_drops_think(self) -> None: + """Hosted APIs have no chat_template_kwargs; think never hits the wire.""" + ref = parse_model_ref("openai/gpt-4o") + result = translate_options({"temperature": 0.5, "think": False}, ref) + assert result == {"temperature": 0.5} + + def test_local_ref_through_sdk_drops_think(self) -> None: + """litellm has no chat_template_kwargs passthrough either.""" + ref = parse_model_ref(_LOCAL_REF) + result = translate_options({"temperature": 0.5, "think": False}, ref) + assert "think" not in result + + +class TestChatOptionsThink: + def test_think_false_maps_to_template_kwargs(self) -> None: + from lilbee.providers.engine_params import chat_options_to_kwargs + + result = chat_options_to_kwargs({"num_predict": 100, "think": False}) + assert result["max_tokens"] == 100 + assert result["chat_template_kwargs"] == {"enable_thinking": False} + assert "think" not in result + + def test_absent_think_adds_no_template_kwargs(self) -> None: + from lilbee.providers.engine_params import chat_options_to_kwargs + + result = chat_options_to_kwargs({"num_predict": 100}) + assert "chat_template_kwargs" not in result + # Options a chat caller can supply; both backends must agree on num_predict and # never emit a key the receiving SDK errors on. diff --git a/tests/test_server_handlers.py b/tests/test_server_handlers.py index 5b1274264..580dd6c74 100644 --- a/tests/test_server_handlers.py +++ b/tests/test_server_handlers.py @@ -239,6 +239,28 @@ async def test_returns_config_and_sources(self): assert result.sources == [] assert result.total_chunks == 0 + async def test_status_carries_entities_section(self): + """The entity section must survive the StatusResponse mapping; a + missing field there silently drops it from the HTTP surface.""" + from lilbee.app.status import EntityStatus, StatusConfig, StatusResult + + mock_status = StatusResult( + config=StatusConfig( + documents_dir="docs", + data_dir="data", + chat_model="test:latest", + embedding_model="embed:latest", + ), + sources=[], + total_chunks=0, + entities=EntityStatus(types=["part_number"], rows=3), + ) + with patch("lilbee.server.handlers.gather_status", return_value=mock_status): + result = await handlers.status() + assert result.entities is not None + assert result.entities.types == ["part_number"] + assert result.entities.rows == 3 + async def test_exposes_all_four_model_roles(self): """/api/status config payload surfaces vision and reranker slots.""" cfg.vision_model = "" From 3e319fb4264e532411a0f6cd2770bb1568e54b74 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:09:27 -0400 Subject: [PATCH 67/77] retrieval: harden quoted-name detection and share noun pluralization Quoted document names now require a matching quote pair detached from surrounding words, so contractions and possessives no longer pair into phantom references, and double-quoted names may contain apostrophes. Noun/type resolution moves to a shared noun_variants helper covering irregular and -ies/-es plurals, replacing two divergent naive pluralizers. Docstrings that still described the abandoned manual schema-review workflow now describe the automatic lifecycle. --- src/lilbee/retrieval/entities/extractor.py | 4 +- src/lilbee/retrieval/entities/schema.py | 67 ++++++++++++++++------ src/lilbee/retrieval/query/intent.py | 11 +++- src/lilbee/retrieval/query/searcher.py | 11 ++-- tests/test_entities_extractor.py | 30 ++++++++++ tests/test_intent.py | 12 ++++ 6 files changed, 107 insertions(+), 28 deletions(-) diff --git a/src/lilbee/retrieval/entities/extractor.py b/src/lilbee/retrieval/entities/extractor.py index 1a0ffcdf4..0c5ea8641 100644 --- a/src/lilbee/retrieval/entities/extractor.py +++ b/src/lilbee/retrieval/entities/extractor.py @@ -1,13 +1,13 @@ """Two-phase typed entity extraction. Phase 1 (cheap): an LLM reads a stratified sample of chunks and proposes the -corpus-specific type schema, persisted as a reviewable artifact before any +corpus-specific type schema, persisted as an editable artifact before any expensive pass runs. Phase 2 (scales with corpus): each type is found by the cheapest extractor that can serve it: compiled regex for identifier-shaped types, spaCy labels for the general ones, and an LLM only for types neither can catch. Cost is -therefore dominated by how many LLM-kind types the reviewed schema keeps. +therefore dominated by how many LLM-kind types the schema keeps. """ from __future__ import annotations diff --git a/src/lilbee/retrieval/entities/schema.py b/src/lilbee/retrieval/entities/schema.py index 83a72743b..a534946c0 100644 --- a/src/lilbee/retrieval/entities/schema.py +++ b/src/lilbee/retrieval/entities/schema.py @@ -1,11 +1,11 @@ """Typed entity extraction: the schema artifact and the entities table. The extraction taxonomy is induced from the corpus, not fixed: a general NER -tag set has no notion of the identifier types a specific corpus carries, so a -schema built from a sample is proposed first, written to a reviewable JSON -artifact, and only then applied at scale. The artifact is the contract: a -human (or agent) can edit types, patterns, and synonyms before paying for the -corpus-wide pass. +tag set has no notion of the identifier types a specific corpus carries. +Sync induces a schema from a corpus sample and applies it automatically; the +JSON artifact it writes is the contract for later tuning: a human (or agent) +can edit types, patterns, and synonyms after the fact, and the next sync +detects the edit and re-applies it across the corpus. """ from __future__ import annotations @@ -52,25 +52,60 @@ def _slugify(cls, v: str) -> str: return slug +# Plural forms the suffix rules below can't produce, mapped both ways. +_IRREGULAR_PLURALS = { + "person": "people", + "man": "men", + "woman": "women", + "child": "children", + "foot": "feet", + "tooth": "teeth", + "mouse": "mice", + "goose": "geese", +} +_IRREGULAR_SINGULARS = {plural: singular for singular, plural in _IRREGULAR_PLURALS.items()} + + +def noun_variants(noun: str) -> set[str]: + """Normalized spelling variants of a noun phrase: itself plus + singular/plural forms of its last word ("tail numbers" ~ "tail number", + "people" ~ "person"). Over-generated junk forms match nothing; a missed + form only fails to resolve, never resolves wrongly. + """ + normalized = " ".join(noun.strip().lower().split()) + if not normalized: + return set() + head, _, last = normalized.rpartition(" ") + prefix = head + " " if head else "" + forms = {last} + if last in _IRREGULAR_PLURALS: + forms.add(_IRREGULAR_PLURALS[last]) + if last in _IRREGULAR_SINGULARS: + forms.add(_IRREGULAR_SINGULARS[last]) + if last.endswith("ies") and len(last) > len("ies"): + forms.add(last[:-3] + "y") + if last.endswith("y"): + forms.add(last[:-1] + "ies") + if last.endswith(("ses", "xes", "zes", "ches", "shes")): + forms.add(last[:-2]) + forms.add(last[:-1] if last.endswith("s") else last + "s") + return {prefix + form for form in forms} + + class EntitySchema(BaseModel): - """The reviewable extraction contract for one corpus.""" + """The editable extraction contract for one corpus.""" types: list[EntityType] def type_for_noun(self, noun: str) -> EntityType | None: """Resolve a question noun (singular or plural) to a type, if any.""" - wanted = noun.strip().lower() - candidates = {wanted} - if wanted.endswith("s"): - candidates.add(wanted[:-1]) - else: - candidates.add(wanted + "s") + candidates = noun_variants(noun) for entity_type in self.types: names = {entity_type.name, entity_type.name.replace("_", " ")} - names.update(s.strip().lower() for s in entity_type.synonyms) - expanded = set(names) - for n in names: - expanded.add(n + "s" if not n.endswith("s") else n[:-1]) + names.update(entity_type.synonyms) + expanded: set[str] = set() + for name in names: + expanded |= noun_variants(name) if candidates & expanded: return entity_type return None diff --git a/src/lilbee/retrieval/query/intent.py b/src/lilbee/retrieval/query/intent.py index 169e177b9..5f83ac950 100644 --- a/src/lilbee/retrieval/query/intent.py +++ b/src/lilbee/retrieval/query/intent.py @@ -63,8 +63,11 @@ class AggregateQuery: re.IGNORECASE, ) -# Quoted names: 'harbor survey 2002' / "harbor survey 2002". -_QUOTED_RE = re.compile(r"[\"']([^\"']{2,80})[\"']") +# Quoted names: 'harbor survey 2002' / "harbor survey 2002". A quote only +# delimits when the pair matches and neither end touches a word from the +# outside, so contractions and possessives ("what's", "Alice's") never pair +# into a phantom name; double-quoted names may contain apostrophes. +_QUOTED_RE = re.compile(r"(? list[str]: for m in _FILENAME_RE.finditer(question): candidates.append(m.group(0).strip()) for m in _QUOTED_RE.finditer(question): - candidates.append(m.group(1).strip()) + quoted = (m.group(1) or m.group(2)).strip() + if quoted: + candidates.append(quoted) for m in _DOC_REF_RE.finditer(question): ref = m.group(1).strip() if ref.lower() not in _REF_STOPWORDS: diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index a096650a4..d6f0d95d8 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -24,6 +24,7 @@ ) from lilbee.providers.base import LLMProvider, ProviderError, ProviderErrorKind from lilbee.retrieval.embedder import Embedder +from lilbee.retrieval.entities.schema import noun_variants from lilbee.retrieval.query.dedup import ( _greedy_cover, _relevance_weight, @@ -122,12 +123,8 @@ def _bm25_confidence(score: float | None) -> float: def _noun_names_type(noun: str, type_name: str) -> bool: """Whether the question's noun IS the type (modulo case/space/plural), as opposed to reaching it through a synonym.""" - wanted = " ".join(noun.strip().lower().split()) - names = {type_name, type_name.replace("_", " ")} - variants = set(names) - for n in names: - variants.add(n + "s" if not n.endswith("s") else n[:-1]) - return wanted in variants + named = noun_variants(type_name) | noun_variants(type_name.replace("_", " ")) + return bool(noun_variants(noun) & named) # RAG mode answer when retrieval finds no usable sources: a grounded refusal @@ -863,7 +860,7 @@ def _decline_aggregate(self) -> str: def _answer_typed_aggregate(self, aggregate: AggregateQuery) -> str | None: """Exact answers over extracted entities, or None when the question's - nouns don't resolve against the reviewed schema.""" + nouns don't resolve against the extraction schema.""" from lilbee.retrieval.entities import load_schema schema = load_schema(self._config.data_dir) diff --git a/tests/test_entities_extractor.py b/tests/test_entities_extractor.py index 11975f749..3b9f2a0c0 100644 --- a/tests/test_entities_extractor.py +++ b/tests/test_entities_extractor.py @@ -16,6 +16,7 @@ normalize_value, save_schema, ) +from lilbee.retrieval.entities.schema import noun_variants def _text_result(text): @@ -67,6 +68,35 @@ def test_type_for_noun_matches_synonyms_and_plurals(self): assert schema.type_for_noun("Part Number") is not None assert schema.type_for_noun("vessels") is None + def test_type_for_noun_matches_irregular_plurals(self): + schema = EntitySchema(types=[PERSON]) + assert schema.type_for_noun("people") is not None + + def test_type_for_noun_matches_ies_plurals(self): + company = EntityType(name="company", kind=ExtractorKind.LLM, description="orgs") + assert EntitySchema(types=[company]).type_for_noun("companies") is not None + + +class TestNounVariants: + def test_inflects_only_the_last_word(self): + assert "tail number" in noun_variants("tail numbers") + assert "tail numbers" in noun_variants("tail number") + + def test_irregular_forms_map_both_ways(self): + assert "people" in noun_variants("person") + assert "person" in noun_variants("people") + + def test_es_and_ies_forms(self): + assert "box" in noun_variants("boxes") + assert "company" in noun_variants("companies") + assert "companies" in noun_variants("company") + + def test_normalizes_case_and_spacing(self): + assert "part number" in noun_variants(" Part Numbers ") + + def test_blank_has_no_variants(self): + assert noun_variants(" ") == set() + class TestInduceSchema: def test_parses_valid_proposal(self): diff --git a/tests/test_intent.py b/tests/test_intent.py index 95bde87d2..cf3aebfaa 100644 --- a/tests/test_intent.py +++ b/tests/test_intent.py @@ -28,6 +28,18 @@ def test_topical_question_yields_nothing(self): assert document_references("which documents mention the harbor?") == [] assert document_references("what do the files say about travel?") == [] + def test_apostrophes_are_not_quotes(self): + # Contractions and possessives must not pair into a phantom quoted + # name ("what's ... Alice's" would otherwise yield "s in Alice"). + assert document_references("what's in Alice's summary about the harbor?") == [] + + def test_quoted_name_may_contain_an_apostrophe(self): + refs = document_references('open "the keeper\'s log" for me') + assert refs == ["the keeper's log"] + + def test_mismatched_quote_characters_do_not_pair(self): + assert document_references("it was labeled \"harbor log' by someone") == [] + def test_generic_noun_after_document_is_not_a_reference(self): assert document_references("is there a document that lists the deliveries?") == [] From 5fab95f0b68de5856b42cdbd1084975e31de5bbc Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:26:17 -0400 Subject: [PATCH 68/77] ask: an empty library answers with add-content guidance, not a failed search With nothing indexed, every question got the grounded refusal ('I couldn't find anything in the indexed documents'), which implies a search ran and came up empty. There is nothing to search, so ask, ask_raw, ask_stream, and the server's SSE stream path now short-circuit to guidance pointing the user at adding documents first. The check lives at the searcher choke point so the TUI, CLI, HTTP, and MCP surfaces all answer consistently, and the copy stays surface-neutral. The store is never searched and the model is never called on the empty path. --- src/lilbee/retrieval/query/searcher.py | 18 +++++++++++++ src/lilbee/server/handlers/rag.py | 11 +++++++- tests/conftest.py | 1 + tests/test_query.py | 35 +++++++++++++++++++++++++- tests/test_server_handlers.py | 18 +++++++++++++ 5 files changed, 81 insertions(+), 2 deletions(-) diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index d6f0d95d8..a3641f037 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -132,6 +132,15 @@ def _noun_names_type(noun: str, type_name: str) -> bool: # off-corpus answers can switch to chat mode. _GROUNDED_REFUSAL = "I couldn't find anything in the indexed documents that answers that." +# Ask/search answer when the library holds nothing yet. Distinct from the +# grounded refusal, which implies a search ran and came up empty: here there is +# nothing to search, so point the user at adding content. Shared across TUI, +# CLI, HTTP, and MCP, so the phrasing stays surface-neutral (no slash commands). +EMPTY_LIBRARY = ( + "Your library is empty, so there's nothing to search yet. " + "Add documents to your library first, then ask again." +) + # Ask/search needs an embedder to ground an answer. When none is loaded, refuse # with an actionable message rather than hard-failing or silently answering # ungrounded; chat mode stays available for an off-corpus reply. @@ -950,6 +959,10 @@ def search_unavailable(self) -> bool: and not self._embedder.embedding_available() ) + def library_empty(self) -> bool: + """Whether the store holds no indexed content yet (nothing to search).""" + return not self._store.has_chunks() + def direct_messages( self, question: str, history: list[ChatMessage] | None = None ) -> list[ChatMessage]: @@ -998,6 +1011,8 @@ def ask_raw( return AskResult(answer=SEARCH_NEEDS_EMBEDDER, sources=[]) if self.skip_retrieval(): return AskResult(answer=self._direct_chat(question, history, options), sources=[]) + if self.library_empty(): + return AskResult(answer=EMPTY_LIBRARY, sources=[]) aggregate_answer = self.route_direct_answer(question) if aggregate_answer is not None: return AskResult(answer=aggregate_answer, sources=[]) @@ -1086,6 +1101,9 @@ def ask_stream( if self.skip_retrieval(): yield from self._stream_direct(question, history, options) return + if self.library_empty(): + yield StreamToken(content=EMPTY_LIBRARY, is_reasoning=False) + return aggregate_answer = self.route_direct_answer(question) if aggregate_answer is not None: yield StreamToken(content=aggregate_answer, is_reasoning=False) diff --git a/src/lilbee/server/handlers/rag.py b/src/lilbee/server/handlers/rag.py index abb233101..9f12fbb3e 100644 --- a/src/lilbee/server/handlers/rag.py +++ b/src/lilbee/server/handlers/rag.py @@ -19,7 +19,7 @@ from lilbee.providers.base import ProviderError, ProviderErrorKind from lilbee.providers.roles import WorkerRole from lilbee.retrieval.query.formatting import cited_subset -from lilbee.retrieval.query.searcher import SEARCH_NEEDS_EMBEDDER +from lilbee.retrieval.query.searcher import EMPTY_LIBRARY, SEARCH_NEEDS_EMBEDDER from lilbee.retrieval.reasoning import ( CAP_CONTINUATION_PROMPT, CAP_NOTICE_TEMPLATE, @@ -561,6 +561,15 @@ def _resolve_stream_context( """ if retrieval_off: return [], searcher.direct_messages(question, history), [] + if searcher.library_empty(): + # Nothing indexed yet: point the user at adding content instead of + # reporting an empty search, matching Searcher.ask_stream. + frames = [ + sse_event(SseEvent.TOKEN, {"token": EMPTY_LIBRARY}), + sse_event(SseEvent.SOURCES, []), + sse_done({}), + ] + return [], None, frames direct = searcher.route_direct_answer(question) if direct is not None: frames = [ diff --git a/tests/conftest.py b/tests/conftest.py index 8e98fc5d9..45d1fc6d4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -339,6 +339,7 @@ def _default_provider_mock(): def _default_store_mock(): store = MagicMock() + store.has_chunks.return_value = True store.search.return_value = [] store.bm25_probe.return_value = [] store.get_sources.return_value = [] diff --git a/tests/test_query.py b/tests/test_query.py index 334a407b0..121fe1c3a 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -24,7 +24,11 @@ _format_citation, cited_subset, ) -from lilbee.retrieval.query.searcher import _GROUNDED_REFUSAL, SEARCH_NEEDS_EMBEDDER +from lilbee.retrieval.query.searcher import ( + _GROUNDED_REFUSAL, + EMPTY_LIBRARY, + SEARCH_NEEDS_EMBEDDER, +) from tests.conftest import make_citation @@ -730,6 +734,17 @@ def test_no_results_returns_grounded_refusal(self, mock_svc): assert result.cited_sources == [] mock_svc.provider.chat.assert_not_called() + def test_empty_library_returns_add_content_guidance(self, mock_svc): + """With nothing indexed, ask_raw points the user at adding content + instead of the grounded refusal (which implies a search happened). The + store is never searched and the model is never called.""" + mock_svc.store.has_chunks.return_value = False + result = get_services().searcher.ask_raw("say hello") + assert result.answer == EMPTY_LIBRARY + assert result.sources == [] + mock_svc.store.search.assert_not_called() + mock_svc.provider.chat.assert_not_called() + def test_ask_raw_with_history(self, mock_svc): mock_svc.store.search.return_value = [_make_result()] mock_svc.provider.chat.return_value = _text_result("answer") @@ -778,6 +793,14 @@ def test_no_results_returns_grounded_refusal(self, mock_svc): assert "Sources:" not in answer mock_svc.provider.chat.assert_not_called() + def test_empty_library_returns_add_content_guidance(self, mock_svc): + """ask() surfaces the empty-library guidance verbatim, no Sources block.""" + mock_svc.store.has_chunks.return_value = False + answer = get_services().searcher.ask("say hello") + assert answer == EMPTY_LIBRARY + assert "Sources:" not in answer + mock_svc.provider.chat.assert_not_called() + def test_ask_with_history(self, mock_svc): mock_svc.store.search.return_value = [_make_result()] mock_svc.provider.chat.return_value = _text_result("answer") @@ -810,6 +833,16 @@ def test_empty_results_streams_grounded_refusal(self, mock_svc): assert "Sources:" not in combined mock_svc.provider.chat.assert_not_called() + def test_empty_library_streams_add_content_guidance(self, mock_svc): + """With nothing indexed, the stream yields the empty-library guidance as a + single token, no Sources block, and never calls the model.""" + mock_svc.store.has_chunks.return_value = False + stream_tokens = list(get_services().searcher.ask_stream("say hello")) + combined = "".join(st.content for st in stream_tokens) + assert combined == EMPTY_LIBRARY + assert "Sources:" not in combined + mock_svc.provider.chat.assert_not_called() + def test_ask_stream_with_history(self, mock_svc): mock_svc.store.search.return_value = [_make_result()] mock_svc.provider.chat.return_value = iter(["response"]) diff --git a/tests/test_server_handlers.py b/tests/test_server_handlers.py index 580dd6c74..a3da5400d 100644 --- a/tests/test_server_handlers.py +++ b/tests/test_server_handlers.py @@ -73,6 +73,8 @@ def mock_svc(): # Default to the grounded retrieval path; mode/embedder tests flip these. searcher.skip_retrieval.return_value = False searcher.search_unavailable.return_value = False + # The library has content unless an empty-library test flips this. + searcher.library_empty.return_value = False # No direct (count-scan) answer unless a test routes one explicitly. searcher.route_direct_answer.return_value = None services = make_mock_services(searcher=searcher) @@ -441,6 +443,22 @@ async def test_forwards_chunk_type_to_build_rag_context(self, mock_svc): pass assert mock_svc.searcher.build_rag_context.call_args.kwargs.get("chunk_type") == "wiki" + async def test_empty_library_streams_add_content_guidance(self, mock_svc): + """With nothing indexed, the ask stream points the user at adding content + as a normal answer token (not an SSE error), and never builds RAG context, + so every ask surface surfaces the empty library the same way.""" + from lilbee.retrieval.query.searcher import EMPTY_LIBRARY + + mock_svc.searcher.library_empty.return_value = True + events = [e async for e in handlers.ask_stream("say hello")] + mock_svc.searcher.build_rag_context.assert_not_called() + non_empty = [e for e in events if e] + event_types = [e.split("\n")[0].replace("event: ", "") for e in non_empty] + assert "error" not in event_types + assert event_types[-1] == "done" + token_event = next(e for e in non_empty if e.startswith("event: token")) + assert json.loads(token_event.split("data: ")[1].strip())["token"] == EMPTY_LIBRARY + async def test_search_mode_no_embedder_refuses(self, mock_svc): """Search mode with no embedder refuses by streaming the refusal as a normal answer token (not an SSE error), mirroring Searcher.ask_stream so the From fab8d26b29b2c3ff2b8ed560677b8c1921cc614d Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:47:23 -0400 Subject: [PATCH 69/77] entities: read the lifecycle's config through the active scope ensure_entities and its digest helpers read the process-global cfg, so under the library API (Lilbee(config=...)) the sync-time lifecycle saw entity_extraction=False and silently skipped induction and extraction, and would have read its schema artifacts from the global data_dir. Read active_config() throughout, matching the rest of the ingest path. --- src/lilbee/retrieval/entities/lifecycle.py | 17 ++++++++++------- tests/test_entities_lifecycle.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/lilbee/retrieval/entities/lifecycle.py b/src/lilbee/retrieval/entities/lifecycle.py index 4f88342c6..69ffc2f3c 100644 --- a/src/lilbee/retrieval/entities/lifecycle.py +++ b/src/lilbee/retrieval/entities/lifecycle.py @@ -20,7 +20,7 @@ import threading from typing import TYPE_CHECKING -from lilbee.core.config import CHUNKS_TABLE, ENTITIES_TABLE, cfg +from lilbee.core.config import CHUNKS_TABLE, ENTITIES_TABLE, active_config from lilbee.retrieval.entities.extractor import ( INDUCTION_SAMPLE_SIZE, extract_entities, @@ -48,14 +48,14 @@ def _schema_digest() -> str | None: - path = schema_path(cfg.data_dir) + path = schema_path(active_config().data_dir) if not path.is_file(): return None return hashlib.sha256(path.read_bytes()).hexdigest() def _applied_digest() -> str | None: - marker = cfg.data_dir / _APPLIED_MARKER + marker = active_config().data_dir / _APPLIED_MARKER if not marker.is_file(): return None return marker.read_text().strip() or None @@ -64,7 +64,7 @@ def _applied_digest() -> str | None: def _record_applied(digest: str | None) -> None: if digest is None: return - (cfg.data_dir / _APPLIED_MARKER).write_text(digest) + (active_config().data_dir / _APPLIED_MARKER).write_text(digest) def ensure_entities(cancel: threading.Event | None = None) -> None: @@ -74,12 +74,15 @@ def ensure_entities(cancel: threading.Event | None = None) -> None: the next sync with a log line, and a cancelled pass leaves the applied marker unset so the next sync redoes the (idempotent) full pass. """ - if not cfg.entity_extraction: + # Read through the scope: under the library API the active config is the + # caller's, not the process-global cfg (which may say the feature is off). + config = active_config() + if not config.entity_extraction: return from lilbee.app.services import get_services store = get_services().store - schema = load_schema(cfg.data_dir) + schema = load_schema(config.data_dir) if schema is None: schema = _induce(store) if schema is None: @@ -104,7 +107,7 @@ def _induce(store: Store) -> EntitySchema | None: if schema is None: log.warning("Entity schema induction produced nothing usable; retrying next sync") return None - path = save_schema(schema, cfg.data_dir) + path = save_schema(schema, active_config().data_dir) log.info( "Induced entity schema (%d types) at %s; edit it to tune, the next sync re-applies", len(schema.types), diff --git a/tests/test_entities_lifecycle.py b/tests/test_entities_lifecycle.py index 95bcc6a75..28f052c92 100644 --- a/tests/test_entities_lifecycle.py +++ b/tests/test_entities_lifecycle.py @@ -134,6 +134,22 @@ def test_edited_schema_reextracts_without_double_counting(self, isolated): assert store.entity_value_counts("part_number") == (1, 1) # not doubled assert store.entity_value_counts("dock") == (1, 1) + def test_scoped_config_governs_lifecycle(self, isolated): + """The library API binds its config via config_scope without touching + the process-global cfg; the lifecycle must read the scoped config or + Lilbee(config=...) silently skips entity extraction.""" + from lilbee.core.config import config_scope + + store, _services = isolated + _index_chunks(store, ["part PX4471"]) + save_schema(_part_schema(), cfg.data_dir) + scoped = cfg.model_copy() + cfg.entity_extraction = False # global says off; the scope says on + scoped.entity_extraction = True + with config_scope(scoped): + ensure_entities() + assert store.entity_value_counts("part_number") == (1, 1) + def test_cancelled_pass_restarts_next_sync(self, isolated): store, _services = isolated _index_chunks(store, ["part PX4471"]) From 8af27f5721a8a4da836c0e65b3bd3b149a0f987c Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:47:23 -0400 Subject: [PATCH 70/77] tests: pin library_empty in the litestar stream-mismatch helper The empty-library short-circuit left this helper's mocked searcher with library_empty unconfigured, so the truthy MagicMock diverted the stream before the embedding mismatch could surface. --- tests/test_server_litestar.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_server_litestar.py b/tests/test_server_litestar.py index c070fbf7a..82146f22f 100644 --- a/tests/test_server_litestar.py +++ b/tests/test_server_litestar.py @@ -657,6 +657,8 @@ async def _collect(): # embedder present (search available, retrieval runs). searcher.search_unavailable.return_value = False searcher.skip_retrieval.return_value = False + # The library has content: the empty-library route must pass. + searcher.library_empty.return_value = False # Not a count question: the direct-answer route must pass. searcher.route_direct_answer.return_value = None searcher.build_rag_context.side_effect = self._mismatch() From 109b5ff194a2cadaee58c976a0799ed7383a8610 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 13 Jul 2026 01:28:04 -0400 Subject: [PATCH 71/77] Force OCR every page via ExtractionConfig.force_ocr, not impossible quality thresholds LILBEE_OCR_FORCE was meant to re-OCR every page, but on xberg rc25 it did nothing for born-digital PDFs. It relied on setting an impossible OcrQualityThresholds floor so every page would fail the text-quality gate and fall through to OCR. rc25 short-circuits any page that already has a clean text layer before that gate is consulted, so the floor never fired and forced runs produced byte-identical native extraction with the GPUs idle. Drive the force flag through ExtractionConfig.force_ocr instead, which overrides the text-layer short-circuit and OCRs every page. Scope it to the vision backend so it stays a GPU re-OCR lever; tesseract and OCR-disabled runs are unchanged and normal runs keep native-first extraction. Add a behavioral regression test that drives real xberg over a born-digital PDF through a counting backend: zero OCR calls unforced, one per page under force. This pins the xberg contract so a future bump cannot silently disable forced re-OCR the way rc25's short-circuit did. --- src/lilbee/data/ingest/extract.py | 28 ++--- tests/test_ingest.py | 178 ++++++++++++++++++++++++++++-- 2 files changed, 180 insertions(+), 26 deletions(-) diff --git a/src/lilbee/data/ingest/extract.py b/src/lilbee/data/ingest/extract.py index 4c44f2fa4..9651346d9 100644 --- a/src/lilbee/data/ingest/extract.py +++ b/src/lilbee/data/ingest/extract.py @@ -34,7 +34,7 @@ ) if TYPE_CHECKING: - from xberg import ExtractedDocument, ExtractionConfig, OcrConfig, OcrQualityThresholds + from xberg import ExtractedDocument, ExtractionConfig, OcrConfig log = logging.getLogger(__name__) @@ -114,28 +114,23 @@ def _ocr_config(ocr_token: str | None) -> OcrConfig: return OcrConfig( backend=OcrBackendName.LILBEE_VISION, backend_options=options, - quality_thresholds=_forced_ocr_thresholds(), ) # xberg requires a non-empty language list (4.x defaulted to English; # 5.x errors on an empty one). cfg.ocr_language is validated non-empty. return OcrConfig(backend=OcrBackendName.TESSERACT, language=list(config.ocr_language)) -def _forced_ocr_thresholds() -> OcrQualityThresholds | None: - """OCR-forcing thresholds when LILBEE_OCR_FORCE=1, else None (xberg defaults). +def _ocr_force_requested() -> bool: + """Whether LILBEE_OCR_FORCE opts every page into vision OCR. - Some scans carry a garbage text layer (whitespace-only or invisible text - objects), so the has-text-layer gate skips OCR and extraction yields zero - chunks. An impossible non-whitespace floor makes every page fail the - quality gate and fall through to OCR -- a targeted-reingest lever, not a - default: normal runs must keep native-first extraction.""" + Born-digital PDFs carry a clean text layer that xberg extracts natively and, + since rc25, short-circuits past the OCR backend entirely -- so a re-OCR run + never touches the vision model. This lever sets ExtractionConfig.force_ocr, + which overrides that short-circuit and OCRs every page. A targeted-reingest + lever, not a default: normal runs keep native-first extraction.""" import os - if os.environ.get("LILBEE_OCR_FORCE", "").strip().lower() not in {"1", "true", "yes"}: - return None - from xberg import OcrQualityThresholds - - return OcrQualityThresholds(min_total_non_whitespace=10**9) + return os.environ.get("LILBEE_OCR_FORCE", "").strip().lower() in {"1", "true", "yes"} def extraction_config(mode: ExtractMode, *, ocr_token: str | None = None) -> ExtractionConfig: @@ -148,16 +143,21 @@ def extraction_config(mode: ExtractMode, *, ocr_token: str | None = None) -> Ext # which lilbee never makes, so it is intentionally not set here.) chunking = build_chunking_config() ocr = _ocr_config(ocr_token) + # force_ocr defeats xberg's text-layer short-circuit so every page reaches the + # vision backend; scoped to the vision path, since it is a GPU re-OCR lever. + force_ocr = _ocr_force_requested() and ocr.backend == OcrBackendName.LILBEE_VISION if mode is ExtractMode.PAGINATED: return ExtractionConfig( chunking=chunking, pages=PageConfig(extract_pages=True, insert_page_markers=False), ocr=ocr, + force_ocr=force_ocr, ) return ExtractionConfig( chunking=chunking, output_format=MARKDOWN_OUTPUT, ocr=ocr, + force_ocr=force_ocr, ) diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 0f2491d8f..a2b369479 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -891,25 +891,28 @@ async def test_cancel_preserves_old_chunks_for_modified_file( mock_svc.store.write_chunks_batch.assert_not_called() -class TestForcedOcrThresholds: - """cfg-independent LILBEE_OCR_FORCE lever for scans with a garbage text layer.""" +class TestOcrForceRequested: + """cfg-independent LILBEE_OCR_FORCE lever that OCRs every page, text layer or not.""" - def test_none_when_env_unset(self, monkeypatch): - from lilbee.data.ingest.extract import _forced_ocr_thresholds + def test_false_when_env_unset(self, monkeypatch): + from lilbee.data.ingest.extract import _ocr_force_requested monkeypatch.delenv("LILBEE_OCR_FORCE", raising=False) - assert _forced_ocr_thresholds() is None + assert _ocr_force_requested() is False @pytest.mark.parametrize("value", ["1", "true", "YES"]) - def test_impossible_floor_when_env_set(self, monkeypatch, value): - from lilbee.data.ingest.extract import _forced_ocr_thresholds + def test_true_when_env_set(self, monkeypatch, value): + from lilbee.data.ingest.extract import _ocr_force_requested monkeypatch.setenv("LILBEE_OCR_FORCE", value) - thresholds = _forced_ocr_thresholds() - assert thresholds is not None - # An unreachable non-whitespace floor fails every page's text-layer gate, - # forcing OCR on scans whose garbage text layer would otherwise skip it. - assert thresholds.min_total_non_whitespace == 10**9 + assert _ocr_force_requested() is True + + @pytest.mark.parametrize("value", ["0", "false", "no", "", " "]) + def test_false_for_non_truthy_values(self, monkeypatch, value): + from lilbee.data.ingest.extract import _ocr_force_requested + + monkeypatch.setenv("LILBEE_OCR_FORCE", value) + assert _ocr_force_requested() is False class TestIngestHelpers: @@ -2689,6 +2692,157 @@ def test_vision_backend_with_token_when_model_set(self, isolated_env): assert config["ocr"].backend == "lilbee-vision" assert config["ocr"].backend_options["req"] == "tok-123" + def test_force_ocr_off_by_default(self, isolated_env, monkeypatch): + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.delenv("LILBEE_OCR_FORCE", raising=False) + cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" + config = extraction_config(ExtractMode.PAGINATED) + assert config.get("force_ocr") is False + + @pytest.mark.parametrize("mode_name", ["PAGINATED", "MARKDOWN"]) + def test_force_ocr_set_when_env_and_vision_model(self, isolated_env, monkeypatch, mode_name): + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.setenv("LILBEE_OCR_FORCE", "1") + cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" + config = extraction_config(getattr(ExtractMode, mode_name)) + assert config["force_ocr"] is True + + def test_force_ocr_ignored_without_vision_model(self, isolated_env, monkeypatch): + # Tesseract path: force is a vision/GPU re-OCR lever, so it stays off here. + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.setenv("LILBEE_OCR_FORCE", "1") + cfg.vision_model = "" + config = extraction_config(ExtractMode.PAGINATED) + assert config["ocr"].backend == "tesseract" + assert config.get("force_ocr") is False + + def test_force_ocr_ignored_when_ocr_disabled(self, isolated_env, monkeypatch): + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.setenv("LILBEE_OCR_FORCE", "1") + cfg.enable_ocr = False + cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" + config = extraction_config(ExtractMode.PAGINATED) + assert config["ocr"].enabled is False + assert config.get("force_ocr") is False + + +class _CountingVisionBackend: + """Minimal xberg custom OCR backend registered as 'lilbee-vision' that records + how many pages were sent to OCR. Used to observe whether force_ocr defeats + xberg's text-layer short-circuit.""" + + def __init__(self) -> None: + self.calls = 0 + + def name(self): + from lilbee.data.ingest.types import OcrBackendName + + return OcrBackendName.LILBEE_VISION + + def version(self): + return "0" + + def supported_languages(self): + return [] + + def supports_language(self, _lang): + return True + + def initialize(self): ... + + def shutdown(self): ... + + def backend_type(self): + from xberg import OcrBackendType + + return OcrBackendType.CUSTOM + + def supports_table_detection(self): + return False + + def supports_document_processing(self): + return False + + def emits_structured_markdown(self): + return False + + def process_image(self, _image_bytes, _config): + from xberg import ExtractedDocument + + from lilbee.data.ingest.types import MARKDOWN_MIME + + self.calls += 1 + return ExtractedDocument(content="OCR-TEXT", mime_type=MARKDOWN_MIME) + + def process_image_file(self, _path, config): + return self.process_image(b"", config) + + def process_document(self, _path, _config): + raise NotImplementedError + + +def _born_digital_pdf() -> bytes: + """A 2-page PDF with a clean native text layer (no OCR needed to read it).""" + import io + + from reportlab.pdfgen import canvas + + buf = io.BytesIO() + c = canvas.Canvas(buf) + for i in range(2): + c.drawString(72, 720, f"Page {i + 1} with a perfectly clean native text layer.") + c.showPage() + c.save() + return buf.getvalue() + + +class TestForceOcrRoutesToBackend: + """Behavioral contract against real xberg: LILBEE_OCR_FORCE must OCR every page + of a born-digital PDF through the registered vision backend, defeating xberg's + text-layer short-circuit. This is the guard that a future xberg bump can't + silently disable forced re-OCR (the way rc25's short-circuit did).""" + + @staticmethod + def _extract(pdf: bytes, config) -> tuple[int, str]: + """Extract under a fake 'lilbee-vision' backend; return (ocr_calls, content).""" + from xberg import register_ocr_backend, unregister_ocr_backend + + from lilbee.data.ingest.types import OcrBackendName + from lilbee.data.xberg_extract import extract_document + + backend = _CountingVisionBackend() + register_ocr_backend(backend) + try: + doc = extract_document(pdf, "application/pdf", filename="born.pdf", config=config) + return backend.calls, doc.content or "" + finally: + unregister_ocr_backend(OcrBackendName.LILBEE_VISION) + + def test_native_text_skips_ocr_without_force(self, isolated_env, monkeypatch): + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.delenv("LILBEE_OCR_FORCE", raising=False) + cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" + config = extraction_config(ExtractMode.PAGINATED) + calls, content = self._extract(_born_digital_pdf(), config) + assert calls == 0 + assert "clean native text layer" in content + + def test_force_ocrs_every_page(self, isolated_env, monkeypatch): + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.setenv("LILBEE_OCR_FORCE", "1") + cfg.vision_model = "org/Test-Vision-GGUF/test-vision-Q4_K_M.gguf" + config = extraction_config(ExtractMode.PAGINATED) + calls, content = self._extract(_born_digital_pdf(), config) + assert calls == 2 # one OCR call per page, text layer notwithstanding + assert "OCR-TEXT" in content + assert "clean native text layer" not in content + class TestChunkAndEmbedPagesEmpty: async def test_empty_page_texts_returns_empty(self): From 4daf0dcf1d63437f92462f432b39c3b8da8121af Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 13 Jul 2026 02:49:32 -0400 Subject: [PATCH 72/77] entities: persist the schema inside the index, not as a file The induced schema is machine state, so it now lives in a single-row _entity_schema table in LanceDB and travels with the index; the entity_schema.json artifact, the applied-digest marker, and the edit-detection machinery are gone. There is nothing to review, edit, or keep adjacent to the database, matching the fully-automatic direction: if induction quality needs improving, the fix belongs in induction. Induction now also collapses duplicate extractors (a small model can propose several names for one pattern, filling the table with identical row sets). A rebuilt index drops the schema with everything else and re-induces on the next sync. --- docs/architecture.md | 14 +-- docs/usage.md | 2 +- src/lilbee/app/status.py | 5 +- src/lilbee/core/config/__init__.py | 2 + src/lilbee/core/config/defaults.py | 1 + src/lilbee/core/config/model.py | 9 +- src/lilbee/data/ingest/pipeline.py | 6 +- src/lilbee/data/store/core.py | 51 ++++++++++- src/lilbee/data/store/schema.py | 10 +++ src/lilbee/data/store/types.py | 3 + src/lilbee/retrieval/entities/__init__.py | 2 - src/lilbee/retrieval/entities/extractor.py | 10 +++ src/lilbee/retrieval/entities/lifecycle.py | 63 ++++---------- src/lilbee/retrieval/entities/schema.py | 48 +++++------ src/lilbee/retrieval/query/searcher.py | 4 +- tests/conftest.py | 2 + tests/test_cli.py | 2 + tests/test_entities_extractor.py | 83 +++++++++++++++--- tests/test_entities_ingest.py | 23 +++-- tests/test_entities_lifecycle.py | 99 ++++++++++------------ tests/test_query.py | 16 ++-- 21 files changed, 277 insertions(+), 178 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index c8e6369b2..0aa4e08a6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -707,19 +707,19 @@ Turning the setting on is the whole interaction; sync runs the lifecycle: 1. **Induction** (first sync after enabling, cheap): an LLM reads a stratified sample of the indexed chunks and proposes the corpus-specific - type schema, saved as `entity_schema.json` next to the index. A general - NER tag set has no notion of the identifier types a specific corpus - carries. The file is a tuning artifact, not a gate: edit a pattern and - the next sync detects the change (by digest) and re-extracts. + type schema, persisted inside the index alongside the tables it governs. + A general NER tag set has no notion of the identifier types a specific + corpus carries. The schema is machine state: nothing to review, edit, or + keep next to the database, and it travels with the index. 2. **Extraction** (same sync, then incrementally): each type is found by the cheapest extractor that serves it: compiled regex for identifier-shaped types, spaCy labels for people/organizations/dates (when the `graph` extra's model is available), and an LLM only for types neither can catch. The full pass reads chunk text from the store (no documents re-ingested, no embeddings recomputed) and always clears prior - rows first, so interrupted passes and schema edits never double-count. - New files extract at ingest. If no chat model is available for - induction, sync logs it and retries next time; nothing fails. + rows first, so an interrupted pass never double-counts. New files + extract at ingest. If no chat model is available for induction, sync + logs it and retries next time; nothing fails. Query-time effects: "how many distinct X" answers with exact distinct counts, and "how many X per Y" groups by chunk co-occurrence, both computed by full diff --git a/docs/usage.md b/docs/usage.md index d7e24201e..594427575 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -784,7 +784,7 @@ reason the defaults are the defaults. | `LILBEE_EXPANSION_GUARDRAILS` | `true` | Filter expansion variants whose embedding drifts too far from the original query | | `LILBEE_EXPANSION_SIMILARITY_THRESHOLD` | `0.5` | Minimum query-variant cosine similarity to survive the guardrail | | `LILBEE_CANDIDATE_MULTIPLIER` | `3` | Vector-only candidate pool as a multiple of top_k, feeding MMR reranking | -| `LILBEE_ENTITY_EXTRACTION` | `false` | Extract typed entities for exact count answers; fully automatic at sync (schema induced on first run, `entity_schema.json` editable to tune) | +| `LILBEE_ENTITY_EXTRACTION` | `false` | Extract typed entities for exact count answers; fully automatic at sync (schema induced on first run and stored inside the index) | | `LILBEE_MIN_RELEVANCE_SCORE` | `0.0` | Abstention floor against the canonical [0, 1] relevance score; when every result falls below it, ask refuses instead of answering from noise | | `LILBEE_HISTORY_REWRITE` | `true` | Condense follow-up questions into standalone retrieval queries using chat history | | `LILBEE_INTENT_ROUTING` | `true` | Route document-name lookups to exact retrieval and count questions to a full-corpus scan | diff --git a/src/lilbee/app/status.py b/src/lilbee/app/status.py index 606f736a1..f400e161a 100644 --- a/src/lilbee/app/status.py +++ b/src/lilbee/app/status.py @@ -150,7 +150,8 @@ def entity_status() -> EntityStatus | None: from lilbee.core.config import ENTITIES_TABLE from lilbee.retrieval.entities import load_schema - schema = load_schema(cfg.data_dir) - table = get_services().store.open_table(ENTITIES_TABLE) + store = get_services().store + schema = load_schema(store) + table = store.open_table(ENTITIES_TABLE) rows = table.count_rows() if table is not None else 0 return EntityStatus(types=[t.name for t in schema.types] if schema else [], rows=rows) diff --git a/src/lilbee/core/config/__init__.py b/src/lilbee/core/config/__init__.py index 2b0b1238a..273bebba8 100644 --- a/src/lilbee/core/config/__init__.py +++ b/src/lilbee/core/config/__init__.py @@ -8,6 +8,7 @@ from .defaults import ( CHUNK_CONCEPTS_TABLE as CHUNK_CONCEPTS_TABLE, ENTITIES_TABLE as ENTITIES_TABLE, + ENTITY_SCHEMA_TABLE as ENTITY_SCHEMA_TABLE, CHUNKS_TABLE as CHUNKS_TABLE, CITATIONS_TABLE as CITATIONS_TABLE, CONCEPT_EDGES_TABLE as CONCEPT_EDGES_TABLE, @@ -49,6 +50,7 @@ "DEFAULT_IGNORE_DIRS", "DEFAULT_NUM_CTX", "ENTITIES_TABLE", + "ENTITY_SCHEMA_TABLE", "MEMORIES_TABLE", "META_TABLE", "PAGE_TEXTS_TABLE", diff --git a/src/lilbee/core/config/defaults.py b/src/lilbee/core/config/defaults.py index 83eefe51a..dff60b4e8 100644 --- a/src/lilbee/core/config/defaults.py +++ b/src/lilbee/core/config/defaults.py @@ -46,6 +46,7 @@ CONCEPT_EDGES_TABLE = "concept_edges" CHUNK_CONCEPTS_TABLE = "chunk_concepts" ENTITIES_TABLE = "entities" +ENTITY_SCHEMA_TABLE = "_entity_schema" # Default URL-exclusion regexes for recursive crawls. Grouped by source # CMS / category. User overrides come from LILBEE_CRAWL_EXCLUDE_PATTERNS diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index 1fe8f85db..25a20a0f7 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -136,10 +136,11 @@ class Config(BaseSettings): # Tesseract fallback wall-clock timeout per file, seconds. 0 = no cap. tesseract_timeout: float = ConfigField(default=60.0, ge=0.0, writable=True) - # Opt-in typed entity extraction at ingest (an entities table for exact - # counting and cross-referencing). Needs a reviewed entity_schema.json in - # data_dir; without one, syncs skip extraction. Off by default: the - # corpus-scale pass costs real compute and most vaults never need it. + # Opt-in typed entity extraction (an entities table for exact counting + # and cross-referencing). Fully automatic: sync induces a schema from the + # corpus and extracts across the index; new files extract at ingest. Off + # by default: the corpus-scale pass costs real compute and most vaults + # never need it. entity_extraction: bool = ConfigField(default=False, writable=True) semantic_chunking: bool = ConfigField(default=False, writable=True) topic_threshold: float = ConfigField(default=0.75, ge=0.0, le=1.0, writable=True) diff --git a/src/lilbee/data/ingest/pipeline.py b/src/lilbee/data/ingest/pipeline.py index 4ab3d076c..1173cb6ee 100644 --- a/src/lilbee/data/ingest/pipeline.py +++ b/src/lilbee/data/ingest/pipeline.py @@ -126,8 +126,8 @@ async def _rebuild_concept_clusters() -> None: async def _build_entity_records(records: list[ChunkRecord], source_name: str) -> list[dict] | None: """Extract typed entities for ingested chunks. None when the mode is off. - Gated twice: the ``entity_extraction`` config flag, and the presence of a - reviewed schema artifact; absent either, syncs cost nothing. Extraction + Gated twice: the ``entity_extraction`` config flag, and a schema already + induced into the index; absent either, syncs cost nothing. Extraction failures degrade to no rows for the file, mirroring concept extraction. """ config = active_config() @@ -135,7 +135,7 @@ async def _build_entity_records(records: list[ChunkRecord], source_name: str) -> return None from lilbee.retrieval.entities import ExtractorKind, extract_entities, load_schema - schema = load_schema(config.data_dir) + schema = load_schema(get_services().store) if schema is None: return None nlp = None diff --git a/src/lilbee/data/store/core.py b/src/lilbee/data/store/core.py index 50c7d8a5b..c5b8385ae 100644 --- a/src/lilbee/data/store/core.py +++ b/src/lilbee/data/store/core.py @@ -18,6 +18,7 @@ CHUNKS_TABLE, CITATIONS_TABLE, ENTITIES_TABLE, + ENTITY_SCHEMA_TABLE, MEMORIES_TABLE, META_TABLE, PAGE_TEXTS_TABLE, @@ -40,8 +41,15 @@ refs_compatible, ) from .ranking import mmr_rerank -from .schema import _citations_schema, _meta_schema, _page_texts_schema, _sources_schema +from .schema import ( + _citations_schema, + _entity_schema_state_schema, + _meta_schema, + _page_texts_schema, + _sources_schema, +) from .types import ( + ENTITY_SCHEMA_DELETE_ALL_PREDICATE, META_DELETE_ALL_PREDICATE, META_SCHEMA_VERSION, READ_CONSISTENCY_INTERVAL, @@ -702,6 +710,47 @@ def add_entities(self, records: list[dict]) -> int: table.add(records) return len(records) + def entity_schema_state(self) -> tuple[str, bool] | None: + """The persisted entity schema as ``(json, applied)``, or ``None``. + + The schema is machine state induced from the corpus and lives inside + the index, so it travels with the data. ``applied`` records whether a + full extraction pass completed under this schema; an interrupted pass + leaves it False and the next sync redoes the (idempotent) pass. + """ + table = self.open_table(ENTITY_SCHEMA_TABLE) + if table is None: + return None + rows = table.search().limit(None).to_list() + if not rows: + return None + # One row by contract; take the newest if a rewrite ever left a stale one. + row = max(rows, key=lambda r: r["updated_at"]) + return str(row["schema_json"]), bool(row["applied"]) + + def save_entity_schema(self, schema_json: str, *, applied: bool) -> None: + """Overwrite the single persisted entity schema row.""" + with self._write_lock(): + db = self.get_db() + table = ensure_table(db, ENTITY_SCHEMA_TABLE, _entity_schema_state_schema()) + _safe_delete_unlocked(table, ENTITY_SCHEMA_DELETE_ALL_PREDICATE) + table.add( + [ + { + "schema_json": schema_json, + "applied": applied, + "updated_at": datetime.now(UTC).isoformat(), + } + ] + ) + + def mark_entity_schema_applied(self) -> None: + """Record that a full extraction pass completed under the stored schema.""" + state = self.entity_schema_state() + if state is None: + return + self.save_entity_schema(state[0], applied=True) + def entity_value_counts(self, entity_type: str) -> tuple[int, int]: """(mentions, distinct normalized values) for one entity type. diff --git a/src/lilbee/data/store/schema.py b/src/lilbee/data/store/schema.py index cda61769e..4778c9744 100644 --- a/src/lilbee/data/store/schema.py +++ b/src/lilbee/data/store/schema.py @@ -16,6 +16,16 @@ def _meta_schema() -> pa.Schema: ) +def _entity_schema_state_schema() -> pa.Schema: + return pa.schema( + [ + pa.field("schema_json", pa.utf8()), + pa.field("applied", pa.bool_()), + pa.field("updated_at", pa.utf8()), + ] + ) + + def _sources_schema() -> pa.Schema: return pa.schema( [ diff --git a/src/lilbee/data/store/types.py b/src/lilbee/data/store/types.py index c1b4cef74..40949a32b 100644 --- a/src/lilbee/data/store/types.py +++ b/src/lilbee/data/store/types.py @@ -84,6 +84,9 @@ class ChunkType(StrEnum): # coupling the deletion to any specific column's value domain. META_DELETE_ALL_PREDICATE = "schema_version IS NOT NULL" +# Same, for the single-row ``_entity_schema`` table. +ENTITY_SCHEMA_DELETE_ALL_PREDICATE = "updated_at IS NOT NULL" + class SearchScope(StrEnum): """What the user wants to search over. diff --git a/src/lilbee/retrieval/entities/__init__.py b/src/lilbee/retrieval/entities/__init__.py index 8b733dba2..12d6428d4 100644 --- a/src/lilbee/retrieval/entities/__init__.py +++ b/src/lilbee/retrieval/entities/__init__.py @@ -12,7 +12,6 @@ ExtractorKind, load_schema, save_schema, - schema_path, ) __all__ = [ @@ -25,5 +24,4 @@ "load_schema", "normalize_value", "save_schema", - "schema_path", ] diff --git a/src/lilbee/retrieval/entities/extractor.py b/src/lilbee/retrieval/entities/extractor.py index 0c5ea8641..5520c3d4e 100644 --- a/src/lilbee/retrieval/entities/extractor.py +++ b/src/lilbee/retrieval/entities/extractor.py @@ -116,6 +116,7 @@ def induce_schema(sample_texts: list[str], provider: LLMProvider) -> EntitySchem log.warning("Entity schema induction returned no parseable JSON") return None types: list[EntityType] = [] + seen_extractors: set[tuple[ExtractorKind, str]] = set() for raw in payload.get("types", []): try: entity_type = EntityType.model_validate(raw) @@ -128,6 +129,15 @@ def induce_schema(sample_texts: list[str], provider: LLMProvider) -> EntitySchem except re.error: log.warning("Dropping induced type %s: bad regex", entity_type.name) continue + # Small models sometimes propose several names for one extractor + # (three types sharing a regex triple the table with identical rows); + # the first name wins. LLM kinds have no pattern, so they dedupe by + # description instead. + key = (entity_type.kind, entity_type.pattern.strip() or entity_type.description.strip()) + if key in seen_extractors: + log.warning("Dropping induced type %s: duplicate extractor", entity_type.name) + continue + seen_extractors.add(key) types.append(entity_type) return EntitySchema(types=types) if types else None diff --git a/src/lilbee/retrieval/entities/lifecycle.py b/src/lilbee/retrieval/entities/lifecycle.py index 69ffc2f3c..e3442f40b 100644 --- a/src/lilbee/retrieval/entities/lifecycle.py +++ b/src/lilbee/retrieval/entities/lifecycle.py @@ -3,19 +3,16 @@ Turning ``entity_extraction`` on is the whole user interaction. Sync does the rest: the first run samples the indexed chunks, induces a schema, and extracts across the whole index; later runs extract new files at ingest -(see ``pipeline._build_entity_records``); and an edited -``entity_schema.json`` is detected by digest and re-applied across the -index on the next sync. The schema file is a manage-after-the-fact -artifact for tuning patterns, never a required step. +(see ``pipeline._build_entity_records``). The schema is machine state +persisted inside the index, so it travels with the data and there is +nothing for the user to manage. Extraction is idempotent: every full pass clears previously extracted -rows first, so an interrupted pass or a schema edit can never -double-count. +rows first, so an interrupted pass can never double-count. """ from __future__ import annotations -import hashlib import logging import threading from typing import TYPE_CHECKING @@ -31,7 +28,6 @@ ExtractorKind, load_schema, save_schema, - schema_path, ) if TYPE_CHECKING: @@ -42,57 +38,34 @@ # Chunks per extraction batch during a full pass; bounds the working set. _BACKFILL_BATCH = 2000 -# Sidecar recording the digest of the last schema applied across the index, -# so an edited entity_schema.json triggers exactly one re-extraction. -_APPLIED_MARKER = "entity_schema.applied" - - -def _schema_digest() -> str | None: - path = schema_path(active_config().data_dir) - if not path.is_file(): - return None - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def _applied_digest() -> str | None: - marker = active_config().data_dir / _APPLIED_MARKER - if not marker.is_file(): - return None - return marker.read_text().strip() or None - - -def _record_applied(digest: str | None) -> None: - if digest is None: - return - (active_config().data_dir / _APPLIED_MARKER).write_text(digest) - def ensure_entities(cancel: threading.Event | None = None) -> None: """Bring extracted entities in line with the schema; no-op when off. Failure never fails the sync: a missing chat model defers induction to - the next sync with a log line, and a cancelled pass leaves the applied - marker unset so the next sync redoes the (idempotent) full pass. + the next sync with a log line, and a cancelled pass leaves the schema + unapplied so the next sync redoes the (idempotent) full pass. """ # Read through the scope: under the library API the active config is the # caller's, not the process-global cfg (which may say the feature is off). - config = active_config() - if not config.entity_extraction: + if not active_config().entity_extraction: return from lilbee.app.services import get_services store = get_services().store - schema = load_schema(config.data_dir) + schema = load_schema(store) if schema is None: + # Never induced (or the persisted row is unreadable): induce afresh. schema = _induce(store) if schema is None: return - elif _schema_digest() == _applied_digest(): - return # up to date; new files were covered at ingest else: - log.info("Entity schema changed; re-extracting entities across the index") + state = store.entity_schema_state() + if state is not None and state[1]: + return # induced and fully applied; new files were covered at ingest + log.info("Entity extraction pass incomplete; redoing the full pass") if _full_pass(store, schema, cancel): - _record_applied(_schema_digest()) + store.mark_entity_schema_applied() def _induce(store: Store) -> EntitySchema | None: @@ -107,12 +80,8 @@ def _induce(store: Store) -> EntitySchema | None: if schema is None: log.warning("Entity schema induction produced nothing usable; retrying next sync") return None - path = save_schema(schema, active_config().data_dir) - log.info( - "Induced entity schema (%d types) at %s; edit it to tune, the next sync re-applies", - len(schema.types), - path, - ) + save_schema(schema, store, applied=False) + log.info("Induced entity schema (%d types); extracting across the index", len(schema.types)) return schema diff --git a/src/lilbee/retrieval/entities/schema.py b/src/lilbee/retrieval/entities/schema.py index a534946c0..57d731f36 100644 --- a/src/lilbee/retrieval/entities/schema.py +++ b/src/lilbee/retrieval/entities/schema.py @@ -1,11 +1,11 @@ -"""Typed entity extraction: the schema artifact and the entities table. +"""Typed entity extraction: the schema model and the entities table. The extraction taxonomy is induced from the corpus, not fixed: a general NER tag set has no notion of the identifier types a specific corpus carries. Sync induces a schema from a corpus sample and applies it automatically; the -JSON artifact it writes is the contract for later tuning: a human (or agent) -can edit types, patterns, and synonyms after the fact, and the next sync -detects the edit and re-applies it across the corpus. +schema is machine state, persisted inside the LanceDB index so it travels +with the data and needs no management. There is nothing to review or edit: +if induction quality needs improving, the fix belongs in induction itself. """ from __future__ import annotations @@ -13,14 +13,15 @@ import json import logging from enum import Enum -from pathlib import Path +from typing import TYPE_CHECKING import pyarrow as pa from pydantic import BaseModel, Field, field_validator -log = logging.getLogger(__name__) +if TYPE_CHECKING: + from lilbee.data.store import Store -SCHEMA_FILENAME = "entity_schema.json" +log = logging.getLogger(__name__) class ExtractorKind(Enum): @@ -111,34 +112,27 @@ def type_for_noun(self, noun: str) -> EntityType | None: return None -def schema_path(data_dir: Path) -> Path: - """Where the corpus's schema artifact lives.""" - return data_dir / SCHEMA_FILENAME - - -def load_schema(data_dir: Path) -> EntitySchema | None: - """Read the schema artifact, or ``None`` when absent or unreadable. +def load_schema(store: Store) -> EntitySchema | None: + """Read the persisted schema from the index, or ``None`` when never induced. - Unreadable is logged, not raised: a hand-edited artifact with a typo - should degrade to "extraction off" rather than break sync. + An unparseable row is logged and read as ``None`` so the lifecycle + re-induces automatically instead of failing sync. """ - path = schema_path(data_dir) - if not path.is_file(): + state = store.entity_schema_state() + if state is None: return None + schema_json, _applied = state try: - return EntitySchema.model_validate_json(path.read_text()) + return EntitySchema.model_validate_json(schema_json) except Exception: - log.warning("Entity schema at %s is unreadable; extraction skipped", path, exc_info=True) + log.warning("Persisted entity schema is unreadable; re-inducing", exc_info=True) return None -def save_schema(schema: EntitySchema, data_dir: Path) -> Path: - """Write the schema artifact (pretty, stable order) and return its path.""" - path = schema_path(data_dir) - path.parent.mkdir(parents=True, exist_ok=True) - payload = schema.model_dump(mode="json") - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - return path +def save_schema(schema: EntitySchema, store: Store, *, applied: bool = False) -> None: + """Persist the schema into the index (stable key order).""" + payload = json.dumps(schema.model_dump(mode="json"), sort_keys=True) + store.save_entity_schema(payload, applied=applied) def _entities_schema(dim_unused: int | None = None) -> pa.Schema: diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index a3641f037..2b0e34373 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -852,7 +852,7 @@ def _decline_aggregate(self) -> str: """The honest no-capability answer, naming what IS countable.""" from lilbee.retrieval.entities import load_schema - schema = load_schema(self._config.data_dir) + schema = load_schema(self._store) if schema is not None and schema.types: countable = ", ".join(sorted(t.name.replace("_", " ") for t in schema.types)) return ( @@ -872,7 +872,7 @@ def _answer_typed_aggregate(self, aggregate: AggregateQuery) -> str | None: nouns don't resolve against the extraction schema.""" from lilbee.retrieval.entities import load_schema - schema = load_schema(self._config.data_dir) + schema = load_schema(self._store) counted = schema.type_for_noun(aggregate.noun) if schema else None if schema is None or counted is None: return None diff --git a/tests/conftest.py b/tests/conftest.py index 45d1fc6d4..6c2fa4434 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -340,6 +340,8 @@ def _default_provider_mock(): def _default_store_mock(): store = MagicMock() store.has_chunks.return_value = True + # No entity schema induced unless a test persists one. + store.entity_schema_state.return_value = None store.search.return_value = [] store.bm25_probe.return_value = [] store.get_sources.return_value = [] diff --git a/tests/test_cli.py b/tests/test_cli.py index cf26ed6b0..b427dd192 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -53,6 +53,8 @@ def mock_svc(): store.bm25_probe.return_value = [] store.get_sources.return_value = [] store.add_chunks.return_value = 0 + # No entity schema induced unless a test persists one. + store.entity_schema_state.return_value = None embedder = MagicMock() embedder.embed.return_value = [0.1] * 768 embedder.embed_batch.return_value = [] diff --git a/tests/test_entities_extractor.py b/tests/test_entities_extractor.py index 3b9f2a0c0..da2e9af58 100644 --- a/tests/test_entities_extractor.py +++ b/tests/test_entities_extractor.py @@ -46,21 +46,57 @@ def test_numeric_leading_zeros(self): assert normalize_value("0") == "0" -class TestSchemaArtifact: - def test_round_trip(self, tmp_path): - schema = EntitySchema(types=[PART, PERSON]) - path = save_schema(schema, tmp_path) - assert path.name == "entity_schema.json" - loaded = load_schema(tmp_path) - assert loaded is not None - assert [t.name for t in loaded.types] == ["part_number", "person"] +class TestSchemaPersistence: + @pytest.fixture() + def store(self, tmp_path): + from lilbee.core.config import cfg + from lilbee.data.store import Store - def test_absent_reads_none(self, tmp_path): - assert load_schema(tmp_path) is None + return Store(cfg.model_copy(update={"lancedb_dir": tmp_path / "lancedb_test"})) - def test_unreadable_reads_none(self, tmp_path): - (tmp_path / "entity_schema.json").write_text("{not json") - assert load_schema(tmp_path) is None + def test_round_trip(self, store): + save_schema(EntitySchema(types=[PART, PERSON]), store) + loaded = load_schema(store) + assert loaded is not None + assert [t.name for t in loaded.types] == ["part_number", "person"] + state = store.entity_schema_state() + assert state is not None + assert state[1] is False # saved but not yet applied + + def test_never_induced_reads_none(self, store): + assert store.entity_schema_state() is None + assert load_schema(store) is None + + def test_unreadable_row_reads_none(self, store): + store.save_entity_schema("{not json", applied=False) + assert load_schema(store) is None + + def test_mark_applied_flips_state(self, store): + save_schema(EntitySchema(types=[PART]), store) + store.mark_entity_schema_applied() + state = store.entity_schema_state() + assert state is not None + assert state[1] is True + + def test_mark_applied_without_schema_is_a_noop(self, store): + store.mark_entity_schema_applied() + assert store.entity_schema_state() is None + + def test_emptied_state_table_reads_none(self, store): + """A state table whose row was deleted reads as never-induced.""" + from lilbee.core.config import ENTITY_SCHEMA_TABLE + from lilbee.data.store.types import ENTITY_SCHEMA_DELETE_ALL_PREDICATE + + save_schema(EntitySchema(types=[PART]), store) + store.clear_table(ENTITY_SCHEMA_TABLE, ENTITY_SCHEMA_DELETE_ALL_PREDICATE) + assert store.entity_schema_state() is None + + def test_save_overwrites_the_single_row(self, store): + save_schema(EntitySchema(types=[PART]), store) + save_schema(EntitySchema(types=[PERSON]), store) + loaded = load_schema(store) + assert loaded is not None + assert [t.name for t in loaded.types] == ["person"] def test_type_for_noun_matches_synonyms_and_plurals(self): schema = EntitySchema(types=[PART]) @@ -122,6 +158,27 @@ def test_parses_valid_proposal(self): assert schema is not None assert [t.name for t in schema.types] == ["part_number", "person"] + def test_duplicate_extractors_collapse_to_the_first_name(self): + """Small models sometimes propose several names for one pattern; + identical extractors would fill the table with identical row sets.""" + provider = MagicMock() + provider.chat.return_value = _text_result( + json.dumps( + { + "types": [ + {"name": "title", "kind": "regex", "pattern": "[A-Z][a-z]+"}, + {"name": "event", "kind": "regex", "pattern": "[A-Z][a-z]+"}, + {"name": "person", "kind": "spacy", "pattern": "PERSON"}, + {"name": "vessel", "kind": "llm", "description": "ships"}, + {"name": "boat", "kind": "llm", "description": "ships"}, + ] + } + ) + ) + schema = induce_schema(["text"], provider) + assert schema is not None + assert [t.name for t in schema.types] == ["title", "person", "vessel"] + def test_drops_bad_regex_keeps_rest(self): provider = MagicMock() provider.chat.return_value = _text_result( diff --git a/tests/test_entities_ingest.py b/tests/test_entities_ingest.py index 46e472a5e..7fbe85337 100644 --- a/tests/test_entities_ingest.py +++ b/tests/test_entities_ingest.py @@ -1,6 +1,7 @@ """Tests for the entity-extraction ingest stage and flush.""" import asyncio +import json from unittest import mock import pytest @@ -9,7 +10,15 @@ from lilbee.core.config import cfg from lilbee.data.ingest.pipeline import _build_entity_records, _flush_entity_rows from lilbee.data.ingest.types import _IngestResult -from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind, save_schema +from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind + + +def _persist_schema(services, schema): + """Stub the mock store's persisted schema state.""" + services.store.entity_schema_state.return_value = ( + json.dumps(schema.model_dump(mode="json")), + True, + ) @pytest.fixture() @@ -64,9 +73,9 @@ def test_no_schema_is_none(self, mock_svc): cfg.entity_extraction = True assert asyncio.run(_build_entity_records(_records(), "a.txt")) is None - def test_extracts_rows_with_reviewed_schema(self, mock_svc): + def test_extracts_rows_with_persisted_schema(self, mock_svc): cfg.entity_extraction = True - save_schema(_part_schema(), cfg.data_dir) + _persist_schema(mock_svc, _part_schema()) rows = asyncio.run(_build_entity_records(_records(), "a.txt")) assert rows is not None assert {r["normalized_value"] for r in rows} == {"px4471", "px9001"} @@ -74,7 +83,7 @@ def test_extracts_rows_with_reviewed_schema(self, mock_svc): def test_extraction_failure_degrades_to_none(self, mock_svc): cfg.entity_extraction = True - save_schema(_part_schema(), cfg.data_dir) + _persist_schema(mock_svc, _part_schema()) with mock.patch( "lilbee.retrieval.entities.extract_entities", side_effect=RuntimeError("boom") ): @@ -97,7 +106,7 @@ def _schema_with(self, kinds): def test_spacy_types_load_the_model_when_available(self, mock_svc): cfg.entity_extraction = True - save_schema(self._schema_with({"spacy"}), cfg.data_dir) + _persist_schema(mock_svc, self._schema_with({"spacy"})) fake_nlp = mock.MagicMock(return_value=mock.MagicMock(ents=[])) with ( mock.patch("lilbee.retrieval.concepts.concepts_available", return_value=True), @@ -109,7 +118,7 @@ def test_spacy_types_load_the_model_when_available(self, mock_svc): def test_spacy_model_import_error_degrades(self, mock_svc, caplog): cfg.entity_extraction = True - save_schema(self._schema_with({"spacy"}), cfg.data_dir) + _persist_schema(mock_svc, self._schema_with({"spacy"})) with ( mock.patch("lilbee.retrieval.concepts.concepts_available", return_value=True), mock.patch( @@ -124,7 +133,7 @@ def test_spacy_model_import_error_degrades(self, mock_svc, caplog): def test_llm_types_fetch_the_provider(self, mock_svc): cfg.entity_extraction = True - save_schema(self._schema_with({"llm"}), cfg.data_dir) + _persist_schema(mock_svc, self._schema_with({"llm"})) mock_svc.provider.chat.return_value = mock.MagicMock(text="{}") rows = asyncio.run(_build_entity_records(_records(), "a.txt")) assert rows is not None diff --git a/tests/test_entities_lifecycle.py b/tests/test_entities_lifecycle.py index 28f052c92..b80bdc836 100644 --- a/tests/test_entities_lifecycle.py +++ b/tests/test_entities_lifecycle.py @@ -10,7 +10,7 @@ from lilbee.core.config import cfg from lilbee.data.store import Store from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind, save_schema -from lilbee.retrieval.entities.lifecycle import _APPLIED_MARKER, ensure_entities +from lilbee.retrieval.entities.lifecycle import ensure_entities @pytest.fixture() @@ -61,6 +61,11 @@ def _part_schema(): ) +def _applied(store) -> bool: + state = store.entity_schema_state() + return state is not None and state[1] + + _INDUCED_JSON = ( '{"types": [{"name": "part_number", "kind": "regex", ' '"pattern": "PX\\\\d{4}", "description": "ids", "synonyms": []}]}' @@ -76,34 +81,34 @@ def test_off_is_a_noop(self, isolated): def test_first_run_induces_and_extracts(self, isolated): """Enabling the setting is the whole interaction: one sync pass - induces the schema, saves it, and extracts across the index.""" + induces the schema, persists it into the index, and extracts.""" store, services = isolated _index_chunks(store, ["part PX4471 shipped", "parts PX9001 and PX4471"]) services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) ensure_entities() - assert (cfg.data_dir / "entity_schema.json").is_file() - assert (cfg.data_dir / _APPLIED_MARKER).is_file() + assert store.entity_schema_state() is not None + assert _applied(store) mentions, distinct = store.entity_value_counts("part_number") assert (mentions, distinct) == (3, 2) def test_nothing_indexed_defers_induction(self, isolated): - _store, services = isolated + store, services = isolated ensure_entities() services.provider.chat.assert_not_called() - assert not (cfg.data_dir / "entity_schema.json").is_file() + assert store.entity_schema_state() is None def test_unusable_induction_retries_next_sync(self, isolated): store, services = isolated _index_chunks(store, ["part PX4471"]) services.provider.chat.return_value = SimpleNamespace(text="no json") ensure_entities() - assert not (cfg.data_dir / "entity_schema.json").is_file() + assert store.entity_schema_state() is None # Next sync with a working model succeeds. services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) ensure_entities() assert store.entity_value_counts("part_number") == (1, 1) - def test_up_to_date_schema_is_a_noop(self, isolated): + def test_applied_schema_is_a_noop(self, isolated): store, services = isolated _index_chunks(store, ["part PX4471"]) services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) @@ -112,27 +117,16 @@ def test_up_to_date_schema_is_a_noop(self, isolated): ensure_entities() full_pass.assert_not_called() - def test_edited_schema_reextracts_without_double_counting(self, isolated): - """The manage-after-the-fact path: edit entity_schema.json, the next - sync detects the digest change and re-runs a replace-semantics pass.""" - store, _services = isolated - _index_chunks(store, ["part PX4471 near dock D-77"]) - save_schema(_part_schema(), cfg.data_dir) + def test_unreadable_persisted_schema_reinduces(self, isolated): + """A corrupt persisted row is machine state gone wrong; the lifecycle + replaces it with a freshly induced schema instead of failing sync.""" + store, services = isolated + _index_chunks(store, ["part PX4471"]) + store.save_entity_schema("{not json", applied=False) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) ensure_entities() + assert _applied(store) assert store.entity_value_counts("part_number") == (1, 1) - # Edit: add a second type. Digest changes; next sync re-applies. - save_schema( - EntitySchema( - types=[ - EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}"), - EntityType(name="dock", kind=ExtractorKind.REGEX, pattern=r"D-\d{2}"), - ] - ), - cfg.data_dir, - ) - ensure_entities() - assert store.entity_value_counts("part_number") == (1, 1) # not doubled - assert store.entity_value_counts("dock") == (1, 1) def test_scoped_config_governs_lifecycle(self, isolated): """The library API binds its config via config_scope without touching @@ -142,7 +136,7 @@ def test_scoped_config_governs_lifecycle(self, isolated): store, _services = isolated _index_chunks(store, ["part PX4471"]) - save_schema(_part_schema(), cfg.data_dir) + save_schema(_part_schema(), store) scoped = cfg.model_copy() cfg.entity_extraction = False # global says off; the scope says on scoped.entity_extraction = True @@ -153,14 +147,27 @@ def test_scoped_config_governs_lifecycle(self, isolated): def test_cancelled_pass_restarts_next_sync(self, isolated): store, _services = isolated _index_chunks(store, ["part PX4471"]) - save_schema(_part_schema(), cfg.data_dir) + save_schema(_part_schema(), store) cancelled = threading.Event() cancelled.set() ensure_entities(cancel=cancelled) - assert not (cfg.data_dir / _APPLIED_MARKER).is_file() + assert not _applied(store) + ensure_entities() + assert _applied(store) + assert store.entity_value_counts("part_number") == (1, 1) + + def test_interrupted_pass_never_double_counts(self, isolated): + """The full pass clears prior rows first, so redoing an unapplied + schema replaces rows instead of appending duplicates.""" + store, _services = isolated + _index_chunks(store, ["part PX4471"]) + save_schema(_part_schema(), store) ensure_entities() - assert (cfg.data_dir / _APPLIED_MARKER).is_file() assert store.entity_value_counts("part_number") == (1, 1) + # Simulate an interrupted pass recorded as unapplied: redo replaces. + save_schema(_part_schema(), store, applied=False) + ensure_entities() + assert store.entity_value_counts("part_number") == (1, 1) # not doubled def test_spacy_and_llm_kinds_wire_their_tools(self, isolated): store, services = isolated @@ -173,7 +180,7 @@ def test_spacy_and_llm_kinds_wire_their_tools(self, isolated): EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}"), ] ), - cfg.data_dir, + store, ) services.provider.chat.return_value = SimpleNamespace(text="{}") fake_nlp = mock.MagicMock(return_value=mock.MagicMock(ents=[])) @@ -196,7 +203,7 @@ def test_spacy_model_import_error_degrades(self, isolated, caplog): EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}"), ] ), - cfg.data_dir, + store, ) with ( mock.patch("lilbee.retrieval.concepts.concepts_available", return_value=True), @@ -211,10 +218,10 @@ def test_spacy_model_import_error_degrades(self, isolated, caplog): assert any("spaCy model unavailable" in r.message for r in caplog.records) def test_empty_chunks_table_skips_pass(self, isolated): - _store, _services = isolated - save_schema(_part_schema(), cfg.data_dir) + store, _services = isolated + save_schema(_part_schema(), store) ensure_entities() - assert not (cfg.data_dir / _APPLIED_MARKER).is_file() + assert not _applied(store) def test_emptied_chunks_table_defers_induction(self, isolated): """A chunks table whose rows were all removed samples nothing.""" @@ -225,24 +232,6 @@ def test_emptied_chunks_table_defers_induction(self, isolated): ensure_entities() services.provider.chat.assert_not_called() - def test_blank_applied_marker_reads_as_unapplied(self, isolated): - """A truncated marker file must trigger a (safe, idempotent) re-pass.""" - store, _services = isolated - _index_chunks(store, ["part PX4471"]) - save_schema(_part_schema(), cfg.data_dir) - (cfg.data_dir / _APPLIED_MARKER).write_text("") - ensure_entities() - assert store.entity_value_counts("part_number") == (1, 1) - assert (cfg.data_dir / _APPLIED_MARKER).read_text().strip() - - def test_missing_schema_digest_never_records_marker(self, isolated): - """_record_applied(None) is the no-op guard for a vanished schema file.""" - from lilbee.retrieval.entities.lifecycle import _record_applied, _schema_digest - - assert _schema_digest() is None - _record_applied(None) - assert not (cfg.data_dir / _APPLIED_MARKER).is_file() - class TestStatusEntities: def test_status_reports_types_and_rows(self, isolated): @@ -250,7 +239,7 @@ def test_status_reports_types_and_rows(self, isolated): store, _services = isolated _index_chunks(store, ["part PX4471"]) - save_schema(_part_schema(), cfg.data_dir) + save_schema(_part_schema(), store) ensure_entities() result = gather_status() assert result.entities is not None diff --git a/tests/test_query.py b/tests/test_query.py index 121fe1c3a..7b41d8873 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1903,8 +1903,10 @@ def test_disabled_by_config_goes_topical(self, mock_svc): class TestTypedAggregates: @pytest.fixture() - def part_schema(self, tmp_path): - from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind, save_schema + def part_schema(self, mock_svc): + import json + + from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind schema = EntitySchema( types=[ @@ -1917,11 +1919,11 @@ def part_schema(self, tmp_path): EntityType(name="depot", kind=ExtractorKind.SPACY, pattern="GPE"), ] ) - old_dir = cfg.data_dir - cfg.data_dir = tmp_path - save_schema(schema, tmp_path) - yield schema - cfg.data_dir = old_dir + mock_svc.store.entity_schema_state.return_value = ( + json.dumps(schema.model_dump(mode="json")), + True, + ) + return schema def test_distinct_count_answers_exactly(self, mock_svc, part_schema): mock_svc.store.entity_value_counts.return_value = (57, 12) From 354a169d8fa0101660fdf3f25665a041137ee45a Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 13 Jul 2026 04:07:26 -0400 Subject: [PATCH 73/77] entities: keep the induced taxonomy up to date as the corpus grows The schema was induced once and frozen, so a library that later grew a new kind of document kept answering with the type set induced on day one: new values of known types flowed in at ingest, but no new type ever appeared. The schema row now records the document count it was induced from, and once the corpus has grown materially past that, sync re-induces from a fresh sample and unions in types the schema has never seen. A type is identified by what it extracts, not the name a model gave it, so a rename of a known pattern adds nothing and induction reuses the same key to collapse duplicate proposals. Re-induction that finds nothing new records the corpus size and skips the pass; one that adds a type re-extracts under the existing replace semantics, so counts never double. --- docs/architecture.md | 11 + docs/usage.md | 2 +- src/lilbee/data/store/core.py | 23 +- src/lilbee/data/store/schema.py | 1 + src/lilbee/data/store/types.py | 16 ++ src/lilbee/retrieval/entities/__init__.py | 6 + src/lilbee/retrieval/entities/extractor.py | 12 +- src/lilbee/retrieval/entities/lifecycle.py | 95 ++++++++- src/lilbee/retrieval/entities/schema.py | 64 ++++-- tests/test_entities_extractor.py | 16 +- tests/test_entities_ingest.py | 10 +- tests/test_entities_lifecycle.py | 235 ++++++++++++++++++++- tests/test_query.py | 10 +- 13 files changed, 440 insertions(+), 61 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 189944f53..650448368 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -806,6 +806,17 @@ Turning the setting on is the whole interaction; sync runs the lifecycle: rows first, so an interrupted pass never double-counts. New files extract at ingest. If no chat model is available for induction, sync logs it and retries next time; nothing fails. +3. **Evolution** (later syncs): a library that grows a new kind of document + would otherwise keep answering with the taxonomy induced on day one, so + the schema row records how many documents the index held at induction, + and once the corpus has grown materially past that (half again as many, + or any growth at all while the corpus is tiny) sync re-induces from a + fresh sample and unions in types the schema has never seen. A type is + identified by what it *extracts*, not the name a model gave it, so a + rename of a known pattern adds nothing. Re-induction that finds nothing + new costs one sampled chat call, records the new size, and skips the + extraction pass; one that adds a type re-extracts under replace + semantics, so counts never double. Query-time effects: "how many distinct X" answers with exact distinct counts, and "how many X per Y" groups by chunk co-occurrence, both computed by full diff --git a/docs/usage.md b/docs/usage.md index f1c700fb2..f18f91459 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -822,7 +822,7 @@ reason the defaults are the defaults. | `LILBEE_EXPANSION_GUARDRAILS` | `true` | Filter expansion variants whose embedding drifts too far from the original query | | `LILBEE_EXPANSION_SIMILARITY_THRESHOLD` | `0.5` | Minimum query-variant cosine similarity to survive the guardrail | | `LILBEE_CANDIDATE_MULTIPLIER` | `3` | Vector-only candidate pool as a multiple of top_k, feeding MMR reranking | -| `LILBEE_ENTITY_EXTRACTION` | `false` | Extract typed entities for exact count answers; fully automatic at sync (schema induced on first run and stored inside the index) | +| `LILBEE_ENTITY_EXTRACTION` | `false` | Extract typed entities for exact count answers; fully automatic at sync (schema induced from the corpus, stored inside the index, and re-induced as the library grows) | | `LILBEE_MIN_RELEVANCE_SCORE` | `0.0` | Abstention floor against the canonical [0, 1] relevance score; when every result falls below it, ask refuses instead of answering from noise | | `LILBEE_HISTORY_REWRITE` | `true` | Condense follow-up questions into standalone retrieval queries using chat history | | `LILBEE_INTENT_ROUTING` | `true` | Route document-name lookups to exact retrieval and count questions to a full-corpus scan | diff --git a/src/lilbee/data/store/core.py b/src/lilbee/data/store/core.py index c5b8385ae..3bf452922 100644 --- a/src/lilbee/data/store/core.py +++ b/src/lilbee/data/store/core.py @@ -58,6 +58,7 @@ ChunkWrite, CitationRecord, EmbeddingModelMismatchError, + EntitySchemaState, MemoryKind, MemoryRow, PageTextRecord, @@ -710,13 +711,11 @@ def add_entities(self, records: list[dict]) -> int: table.add(records) return len(records) - def entity_schema_state(self) -> tuple[str, bool] | None: - """The persisted entity schema as ``(json, applied)``, or ``None``. + def entity_schema_state(self) -> EntitySchemaState | None: + """The persisted entity schema row, or ``None`` when never induced. The schema is machine state induced from the corpus and lives inside - the index, so it travels with the data. ``applied`` records whether a - full extraction pass completed under this schema; an interrupted pass - leaves it False and the next sync redoes the (idempotent) pass. + the index, so it travels with the data. """ table = self.open_table(ENTITY_SCHEMA_TABLE) if table is None: @@ -726,9 +725,14 @@ def entity_schema_state(self) -> tuple[str, bool] | None: return None # One row by contract; take the newest if a rewrite ever left a stale one. row = max(rows, key=lambda r: r["updated_at"]) - return str(row["schema_json"]), bool(row["applied"]) + return EntitySchemaState( + schema_json=str(row["schema_json"]), + applied=bool(row["applied"]), + source_count=int(row["source_count"]), + updated_at=str(row["updated_at"]), + ) - def save_entity_schema(self, schema_json: str, *, applied: bool) -> None: + def save_entity_schema(self, schema_json: str, *, applied: bool, source_count: int) -> None: """Overwrite the single persisted entity schema row.""" with self._write_lock(): db = self.get_db() @@ -739,6 +743,7 @@ def save_entity_schema(self, schema_json: str, *, applied: bool) -> None: { "schema_json": schema_json, "applied": applied, + "source_count": source_count, "updated_at": datetime.now(UTC).isoformat(), } ] @@ -749,7 +754,9 @@ def mark_entity_schema_applied(self) -> None: state = self.entity_schema_state() if state is None: return - self.save_entity_schema(state[0], applied=True) + self.save_entity_schema( + state["schema_json"], applied=True, source_count=state["source_count"] + ) def entity_value_counts(self, entity_type: str) -> tuple[int, int]: """(mentions, distinct normalized values) for one entity type. diff --git a/src/lilbee/data/store/schema.py b/src/lilbee/data/store/schema.py index 4778c9744..21bd15324 100644 --- a/src/lilbee/data/store/schema.py +++ b/src/lilbee/data/store/schema.py @@ -21,6 +21,7 @@ def _entity_schema_state_schema() -> pa.Schema: [ pa.field("schema_json", pa.utf8()), pa.field("applied", pa.bool_()), + pa.field("source_count", pa.int32()), pa.field("updated_at", pa.utf8()), ] ) diff --git a/src/lilbee/data/store/types.py b/src/lilbee/data/store/types.py index 40949a32b..6ff0829c8 100644 --- a/src/lilbee/data/store/types.py +++ b/src/lilbee/data/store/types.py @@ -88,6 +88,22 @@ class ChunkType(StrEnum): ENTITY_SCHEMA_DELETE_ALL_PREDICATE = "updated_at IS NOT NULL" +class EntitySchemaState(TypedDict): + """Single-row state of the induced entity schema. + + ``applied`` records whether a full extraction pass completed under this + schema; an interrupted pass leaves it False so the next sync redoes the + (idempotent) pass. ``source_count`` is how many documents the index held + when the schema was induced, which is what the next sync compares against + to decide the corpus has drifted far enough to re-induce. + """ + + schema_json: str + applied: bool + source_count: int + updated_at: str + + class SearchScope(StrEnum): """What the user wants to search over. diff --git a/src/lilbee/retrieval/entities/__init__.py b/src/lilbee/retrieval/entities/__init__.py index 12d6428d4..dedfe7845 100644 --- a/src/lilbee/retrieval/entities/__init__.py +++ b/src/lilbee/retrieval/entities/__init__.py @@ -10,7 +10,10 @@ EntitySchema, EntityType, ExtractorKind, + extractor_key, load_schema, + merge_schemas, + parse_schema, save_schema, ) @@ -20,8 +23,11 @@ "ExtractorKind", "ensure_entities", "extract_entities", + "extractor_key", "induce_schema", "load_schema", + "merge_schemas", "normalize_value", + "parse_schema", "save_schema", ] diff --git a/src/lilbee/retrieval/entities/extractor.py b/src/lilbee/retrieval/entities/extractor.py index 5520c3d4e..00149c2a4 100644 --- a/src/lilbee/retrieval/entities/extractor.py +++ b/src/lilbee/retrieval/entities/extractor.py @@ -18,7 +18,12 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any -from lilbee.retrieval.entities.schema import EntitySchema, EntityType, ExtractorKind +from lilbee.retrieval.entities.schema import ( + EntitySchema, + EntityType, + ExtractorKind, + extractor_key, +) from lilbee.retrieval.reasoning import strip_reasoning if TYPE_CHECKING: @@ -131,9 +136,8 @@ def induce_schema(sample_texts: list[str], provider: LLMProvider) -> EntitySchem continue # Small models sometimes propose several names for one extractor # (three types sharing a regex triple the table with identical rows); - # the first name wins. LLM kinds have no pattern, so they dedupe by - # description instead. - key = (entity_type.kind, entity_type.pattern.strip() or entity_type.description.strip()) + # the first name wins. + key = extractor_key(entity_type) if key in seen_extractors: log.warning("Dropping induced type %s: duplicate extractor", entity_type.name) continue diff --git a/src/lilbee/retrieval/entities/lifecycle.py b/src/lilbee/retrieval/entities/lifecycle.py index e3442f40b..b1e72e93f 100644 --- a/src/lilbee/retrieval/entities/lifecycle.py +++ b/src/lilbee/retrieval/entities/lifecycle.py @@ -7,8 +7,16 @@ persisted inside the index, so it travels with the data and there is nothing for the user to manage. +The taxonomy keeps up with the corpus on its own. A library that grows a +new kind of document would otherwise keep answering with the type set +induced on day one, so sync re-induces from a fresh sample once the corpus +has grown materially since the schema was written, and unions in any type +the schema has never seen. Re-induction that proposes nothing new costs one +sampled chat call and no extraction pass. + Extraction is idempotent: every full pass clears previously extracted -rows first, so an interrupted pass can never double-count. +rows first, so an interrupted pass, or a schema that gained a type, can +never double-count. """ from __future__ import annotations @@ -26,18 +34,31 @@ from lilbee.retrieval.entities.schema import ( EntitySchema, ExtractorKind, - load_schema, + merge_schemas, + parse_schema, save_schema, ) if TYPE_CHECKING: from lilbee.data.store import Store + from lilbee.data.store.types import EntitySchemaState log = logging.getLogger(__name__) # Chunks per extraction batch during a full pass; bounds the working set. _BACKFILL_BATCH = 2000 +# Corpus growth that makes the induced taxonomy worth revisiting, as a +# multiple of the document count at induction. Half again as many documents +# is enough to have introduced a family the first sample never saw; smaller +# ratios would re-induce on ordinary drip-feed additions, whose new files are +# already extracted at ingest under the existing schema. +_REINDUCE_GROWTH_FACTOR = 1.5 +# Below this many documents, growth ratios are meaningless (2 -> 3 documents +# is 1.5x): a corpus this small re-induces on any growth, since the first +# sample necessarily saw very little. +_REINDUCE_MIN_SOURCES = 10 + def ensure_entities(cancel: threading.Event | None = None) -> None: """Bring extracted entities in line with the schema; no-op when off. @@ -53,21 +74,77 @@ def ensure_entities(cancel: threading.Event | None = None) -> None: from lilbee.app.services import get_services store = get_services().store - schema = load_schema(store) - if schema is None: + # One read of the schema row: it carries the taxonomy, whether a full + # pass ever completed under it, and the corpus size it was induced from. + state = store.entity_schema_state() + schema = parse_schema(state) if state is not None else None + if state is None or schema is None: # Never induced (or the persisted row is unreadable): induce afresh. schema = _induce(store) if schema is None: return else: - state = store.entity_schema_state() - if state is not None and state[1]: - return # induced and fully applied; new files were covered at ingest - log.info("Entity extraction pass incomplete; redoing the full pass") + evolved = _evolve(store, schema, state) + if evolved is not None: + schema = evolved + elif state["applied"]: + return # up to date; new files were covered at ingest + else: + log.info("Entity extraction pass incomplete; redoing the full pass") if _full_pass(store, schema, cancel): store.mark_entity_schema_applied() +def _evolve(store: Store, schema: EntitySchema, state: EntitySchemaState) -> EntitySchema | None: + """The schema grown to cover a drifted corpus, or ``None`` to leave it be. + + Returns a schema only when re-induction actually found a type the + current one lacks, so a corpus that grew without changing character + costs one chat call and no extraction pass. The new document count is + recorded either way, so a re-induction that adds nothing does not run + again on the next sync. + """ + from lilbee.app.services import get_services + + sources = store.count_sources() + at_induction = state["source_count"] + if not _corpus_drifted(at_induction, sources): + return None + # A drifted corpus has chunks to sample by construction; an empty sample + # would fall through to induce_schema, which declines it like any other + # unusable input, so it needs no separate branch here. + texts = _sample_chunks(store, INDUCTION_SAMPLE_SIZE) + induced = induce_schema(texts, get_services().provider) + if induced is None: + log.warning("Entity schema re-induction produced nothing usable; retrying next sync") + return None + merged = merge_schemas(schema, induced) + if len(merged.types) == len(schema.types): + # Nothing new: record the corpus size so this does not re-run, and + # leave the applied flag alone (extraction is already current). + save_schema(schema, store, applied=state["applied"], source_count=sources) + log.info("Corpus grew but the entity taxonomy still fits; no re-extraction") + return None + added = [t.name for t in merged.types[len(schema.types) :]] + log.info( + "Corpus grew from %d to %d documents; entity schema gained %s. Re-extracting", + at_induction, + sources, + ", ".join(added), + ) + save_schema(merged, store, applied=False, source_count=sources) + return merged + + +def _corpus_drifted(at_induction: int, now: int) -> bool: + """Whether the corpus has grown enough to revisit the taxonomy.""" + if now <= at_induction: + return False + if at_induction < _REINDUCE_MIN_SOURCES: + return True + return now >= at_induction * _REINDUCE_GROWTH_FACTOR + + def _induce(store: Store) -> EntitySchema | None: """First-run schema induction from the indexed chunks. None defers.""" from lilbee.app.services import get_services @@ -80,7 +157,7 @@ def _induce(store: Store) -> EntitySchema | None: if schema is None: log.warning("Entity schema induction produced nothing usable; retrying next sync") return None - save_schema(schema, store, applied=False) + save_schema(schema, store, applied=False, source_count=store.count_sources()) log.info("Induced entity schema (%d types); extracting across the index", len(schema.types)) return schema diff --git a/src/lilbee/retrieval/entities/schema.py b/src/lilbee/retrieval/entities/schema.py index 57d731f36..8a216bdc6 100644 --- a/src/lilbee/retrieval/entities/schema.py +++ b/src/lilbee/retrieval/entities/schema.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: from lilbee.data.store import Store + from lilbee.data.store.types import EntitySchemaState log = logging.getLogger(__name__) @@ -112,27 +113,66 @@ def type_for_noun(self, noun: str) -> EntityType | None: return None -def load_schema(store: Store) -> EntitySchema | None: - """Read the persisted schema from the index, or ``None`` when never induced. +def extractor_key(entity_type: EntityType) -> tuple[ExtractorKind, str]: + """What a type actually extracts, ignoring the name it was given. - An unparseable row is logged and read as ``None`` so the lifecycle - re-induces automatically instead of failing sync. + Two types with the same kind and the same pattern (or, for LLM kinds, + the same description) find the same mentions, so they are the same type + wearing different names. Induction dedupes by this, and schema evolution + uses it to tell a genuinely new type from a rename of a known one. + """ + signature = entity_type.pattern.strip() or entity_type.description.strip() + return entity_type.kind, signature.casefold() + + +def merge_schemas(existing: EntitySchema, induced: EntitySchema) -> EntitySchema: + """Existing types plus any genuinely new ones from *induced*. + + Union, not replace: existing types keep their names and positions so + answers stay stable across a re-induction, and a type the fresh sample + happens not to cover is never silently dropped (the documents it was + induced from are still in the index). Only extractors the schema has + never seen are appended. + """ + known = {extractor_key(t) for t in existing.types} + known_names = {t.name for t in existing.types} + additions = [ + t for t in induced.types if extractor_key(t) not in known and t.name not in known_names + ] + if not additions: + return existing + return EntitySchema(types=[*existing.types, *additions]) + + +def parse_schema(state: EntitySchemaState) -> EntitySchema | None: + """The schema carried by a persisted row, or ``None`` when unreadable. + + Unreadable is logged, not raised: a corrupt row is machine state gone + wrong, so the lifecycle re-induces automatically instead of failing sync. """ - state = store.entity_schema_state() - if state is None: - return None - schema_json, _applied = state try: - return EntitySchema.model_validate_json(schema_json) + return EntitySchema.model_validate_json(state["schema_json"]) except Exception: log.warning("Persisted entity schema is unreadable; re-inducing", exc_info=True) return None -def save_schema(schema: EntitySchema, store: Store, *, applied: bool = False) -> None: - """Persist the schema into the index (stable key order).""" +def load_schema(store: Store) -> EntitySchema | None: + """Read the persisted schema from the index, or ``None`` when never induced.""" + state = store.entity_schema_state() + return parse_schema(state) if state is not None else None + + +def save_schema(schema: EntitySchema, store: Store, *, applied: bool, source_count: int) -> None: + """Persist the schema into the index (stable key order). + + ``source_count`` is the document count the schema was induced from, and + is the baseline the next sync compares against to decide the corpus has + drifted; it is required so a save can never silently record a baseline + of zero, which would read as a tiny corpus and re-induce every sync. + """ payload = json.dumps(schema.model_dump(mode="json"), sort_keys=True) - store.save_entity_schema(payload, applied=applied) + store.save_entity_schema(payload, applied=applied, source_count=source_count) def _entities_schema(dim_unused: int | None = None) -> pa.Schema: diff --git a/tests/test_entities_extractor.py b/tests/test_entities_extractor.py index da2e9af58..d97d1435a 100644 --- a/tests/test_entities_extractor.py +++ b/tests/test_entities_extractor.py @@ -55,28 +55,28 @@ def store(self, tmp_path): return Store(cfg.model_copy(update={"lancedb_dir": tmp_path / "lancedb_test"})) def test_round_trip(self, store): - save_schema(EntitySchema(types=[PART, PERSON]), store) + save_schema(EntitySchema(types=[PART, PERSON]), store, applied=False, source_count=1) loaded = load_schema(store) assert loaded is not None assert [t.name for t in loaded.types] == ["part_number", "person"] state = store.entity_schema_state() assert state is not None - assert state[1] is False # saved but not yet applied + assert state["applied"] is False # saved but not yet applied def test_never_induced_reads_none(self, store): assert store.entity_schema_state() is None assert load_schema(store) is None def test_unreadable_row_reads_none(self, store): - store.save_entity_schema("{not json", applied=False) + store.save_entity_schema("{not json", applied=False, source_count=1) assert load_schema(store) is None def test_mark_applied_flips_state(self, store): - save_schema(EntitySchema(types=[PART]), store) + save_schema(EntitySchema(types=[PART]), store, applied=False, source_count=1) store.mark_entity_schema_applied() state = store.entity_schema_state() assert state is not None - assert state[1] is True + assert state["applied"] is True def test_mark_applied_without_schema_is_a_noop(self, store): store.mark_entity_schema_applied() @@ -87,13 +87,13 @@ def test_emptied_state_table_reads_none(self, store): from lilbee.core.config import ENTITY_SCHEMA_TABLE from lilbee.data.store.types import ENTITY_SCHEMA_DELETE_ALL_PREDICATE - save_schema(EntitySchema(types=[PART]), store) + save_schema(EntitySchema(types=[PART]), store, applied=False, source_count=1) store.clear_table(ENTITY_SCHEMA_TABLE, ENTITY_SCHEMA_DELETE_ALL_PREDICATE) assert store.entity_schema_state() is None def test_save_overwrites_the_single_row(self, store): - save_schema(EntitySchema(types=[PART]), store) - save_schema(EntitySchema(types=[PERSON]), store) + save_schema(EntitySchema(types=[PART]), store, applied=False, source_count=1) + save_schema(EntitySchema(types=[PERSON]), store, applied=False, source_count=1) loaded = load_schema(store) assert loaded is not None assert [t.name for t in loaded.types] == ["person"] diff --git a/tests/test_entities_ingest.py b/tests/test_entities_ingest.py index 7fbe85337..b13408e9d 100644 --- a/tests/test_entities_ingest.py +++ b/tests/test_entities_ingest.py @@ -15,10 +15,12 @@ def _persist_schema(services, schema): """Stub the mock store's persisted schema state.""" - services.store.entity_schema_state.return_value = ( - json.dumps(schema.model_dump(mode="json")), - True, - ) + services.store.entity_schema_state.return_value = { + "schema_json": json.dumps(schema.model_dump(mode="json")), + "applied": True, + "source_count": 1, + "updated_at": "2026-01-01T00:00:00+00:00", + } @pytest.fixture() diff --git a/tests/test_entities_lifecycle.py b/tests/test_entities_lifecycle.py index b80bdc836..d50a4175d 100644 --- a/tests/test_entities_lifecycle.py +++ b/tests/test_entities_lifecycle.py @@ -9,7 +9,13 @@ import lilbee.app.services as svc_mod from lilbee.core.config import cfg from lilbee.data.store import Store -from lilbee.retrieval.entities import EntitySchema, EntityType, ExtractorKind, save_schema +from lilbee.retrieval.entities import ( + EntitySchema, + EntityType, + ExtractorKind, + load_schema, + save_schema, +) from lilbee.retrieval.entities.lifecycle import ensure_entities @@ -34,12 +40,12 @@ def isolated(tmp_path): setattr(cfg, name, getattr(snapshot, name)) -def _index_chunks(store, texts): +def _index_chunks(store, texts, source="catalog.txt"): dim = cfg.embedding_dim store.add_chunks( [ { - "source": "catalog.txt", + "source": source, "content_type": "text", "chunk_type": "raw", "page_start": 1, @@ -55,6 +61,14 @@ def _index_chunks(store, texts): ) +def _index_sources(store, count, *, start=0, text="part PX4471"): + """Index *count* distinct sources so count_sources() sees a real corpus.""" + for i in range(start, start + count): + name = f"doc{i}.txt" + _index_chunks(store, [text], source=name) + store.upsert_source(name, f"hash{i}", chunk_count=1) + + def _part_schema(): return EntitySchema( types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}")] @@ -63,7 +77,7 @@ def _part_schema(): def _applied(store) -> bool: state = store.entity_schema_state() - return state is not None and state[1] + return state is not None and state["applied"] _INDUCED_JSON = ( @@ -71,6 +85,14 @@ def _applied(store) -> bool: '"pattern": "PX\\\\d{4}", "description": "ids", "synonyms": []}]}' ) +# A re-induction that proposes the known part_number type plus a new dock type. +_REINDUCED_JSON = ( + '{"types": [{"name": "part_number", "kind": "regex", ' + '"pattern": "PX\\\\d{4}", "description": "ids", "synonyms": []}, ' + '{"name": "dock", "kind": "regex", "pattern": "D-\\\\d{2}", ' + '"description": "docks", "synonyms": []}]}' +) + class TestEnsureEntities: def test_off_is_a_noop(self, isolated): @@ -122,7 +144,7 @@ def test_unreadable_persisted_schema_reinduces(self, isolated): replaces it with a freshly induced schema instead of failing sync.""" store, services = isolated _index_chunks(store, ["part PX4471"]) - store.save_entity_schema("{not json", applied=False) + store.save_entity_schema("{not json", applied=False, source_count=1) services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) ensure_entities() assert _applied(store) @@ -136,7 +158,7 @@ def test_scoped_config_governs_lifecycle(self, isolated): store, _services = isolated _index_chunks(store, ["part PX4471"]) - save_schema(_part_schema(), store) + save_schema(_part_schema(), store, applied=False, source_count=1) scoped = cfg.model_copy() cfg.entity_extraction = False # global says off; the scope says on scoped.entity_extraction = True @@ -147,7 +169,7 @@ def test_scoped_config_governs_lifecycle(self, isolated): def test_cancelled_pass_restarts_next_sync(self, isolated): store, _services = isolated _index_chunks(store, ["part PX4471"]) - save_schema(_part_schema(), store) + save_schema(_part_schema(), store, applied=False, source_count=1) cancelled = threading.Event() cancelled.set() ensure_entities(cancel=cancelled) @@ -161,11 +183,11 @@ def test_interrupted_pass_never_double_counts(self, isolated): schema replaces rows instead of appending duplicates.""" store, _services = isolated _index_chunks(store, ["part PX4471"]) - save_schema(_part_schema(), store) + save_schema(_part_schema(), store, applied=False, source_count=1) ensure_entities() assert store.entity_value_counts("part_number") == (1, 1) # Simulate an interrupted pass recorded as unapplied: redo replaces. - save_schema(_part_schema(), store, applied=False) + save_schema(_part_schema(), store, applied=False, source_count=1) ensure_entities() assert store.entity_value_counts("part_number") == (1, 1) # not doubled @@ -181,6 +203,8 @@ def test_spacy_and_llm_kinds_wire_their_tools(self, isolated): ] ), store, + applied=False, + source_count=1, ) services.provider.chat.return_value = SimpleNamespace(text="{}") fake_nlp = mock.MagicMock(return_value=mock.MagicMock(ents=[])) @@ -204,6 +228,8 @@ def test_spacy_model_import_error_degrades(self, isolated, caplog): ] ), store, + applied=False, + source_count=1, ) with ( mock.patch("lilbee.retrieval.concepts.concepts_available", return_value=True), @@ -219,7 +245,7 @@ def test_spacy_model_import_error_degrades(self, isolated, caplog): def test_empty_chunks_table_skips_pass(self, isolated): store, _services = isolated - save_schema(_part_schema(), store) + save_schema(_part_schema(), store, applied=False, source_count=1) ensure_entities() assert not _applied(store) @@ -233,13 +259,200 @@ def test_emptied_chunks_table_defers_induction(self, isolated): services.provider.chat.assert_not_called() +class TestSchemaEvolution: + """The taxonomy keeps up with the corpus without anyone touching it: a + library that grows a new kind of document gains the types to answer about + it, and one that merely grows costs a sample and no re-extraction.""" + + def test_new_document_family_adds_types_and_reextracts(self, isolated): + store, services = isolated + _index_sources(store, 12) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + assert [t.name for t in load_schema(store).types] == ["part_number"] + assert store.entity_value_counts("dock") == (0, 0) + + # The corpus grows past the drift threshold with documents carrying a + # kind of identifier the first sample never saw. + _index_sources(store, 8, start=12, text="part PX9001 near dock D-77") + services.provider.chat.return_value = SimpleNamespace(text=_REINDUCED_JSON) + ensure_entities() + + schema = load_schema(store) + assert [t.name for t in schema.types] == ["part_number", "dock"] + assert store.entity_value_counts("dock") == (8, 1) # the new type was extracted + assert _applied(store) + state = store.entity_schema_state() + assert state["source_count"] == 20 # drift baseline moved to the new size + + def test_growth_without_new_types_skips_reextraction(self, isolated): + """Re-induction that proposes nothing new must not pay for a pass.""" + store, services = isolated + _index_sources(store, 12) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + _index_sources(store, 8, start=12) + + with mock.patch("lilbee.retrieval.entities.lifecycle._full_pass") as full_pass: + ensure_entities() + full_pass.assert_not_called() + assert [t.name for t in load_schema(store).types] == ["part_number"] + # The new size is recorded, so the next sync does not re-induce again. + assert store.entity_schema_state()["source_count"] == 20 + with mock.patch("lilbee.retrieval.entities.lifecycle.induce_schema") as induce: + ensure_entities() + induce.assert_not_called() + + def test_ordinary_growth_does_not_reinduce(self, isolated): + """A drip-feed of documents stays under the threshold: those files are + already extracted at ingest under the existing schema.""" + store, services = isolated + _index_sources(store, 12) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + _index_sources(store, 3, start=12) # 15 < 12 * 1.5 + + with mock.patch("lilbee.retrieval.entities.lifecycle.induce_schema") as induce: + ensure_entities() + induce.assert_not_called() + + def test_small_corpus_reinduces_on_any_growth(self, isolated): + """Under the floor a growth ratio is meaningless, and the first sample + necessarily saw very little, so any new document revisits the schema.""" + store, services = isolated + _index_sources(store, 2) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + _index_sources(store, 1, start=2, text="dock D-77") + services.provider.chat.return_value = SimpleNamespace(text=_REINDUCED_JSON) + ensure_entities() + assert [t.name for t in load_schema(store).types] == ["part_number", "dock"] + + def test_shrinking_corpus_never_reinduces(self, isolated): + store, services = isolated + _index_sources(store, 12) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + store.remove_documents(["doc0.txt", "doc1.txt"]) + with mock.patch("lilbee.retrieval.entities.lifecycle.induce_schema") as induce: + ensure_entities() + induce.assert_not_called() + + def test_unusable_reinduction_leaves_the_schema_intact(self, isolated): + """A failed re-induction must not lose the working taxonomy, and must + not record the new size (so the next sync retries).""" + store, services = isolated + _index_sources(store, 12) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + _index_sources(store, 8, start=12) + services.provider.chat.return_value = SimpleNamespace(text="no json") + ensure_entities() + assert [t.name for t in load_schema(store).types] == ["part_number"] + assert store.entity_schema_state()["source_count"] == 12 # retry next sync + + def test_reinduction_ignores_renames_of_known_extractors(self, isolated): + """A model that renames a known pattern must not double the taxonomy: + the type is the extractor, not the label it was given this time.""" + store, services = isolated + _index_sources(store, 12) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + _index_sources(store, 8, start=12) + renamed = _INDUCED_JSON.replace('"part_number"', '"catalog_id"') + services.provider.chat.return_value = SimpleNamespace(text=renamed) + ensure_entities() + assert [t.name for t in load_schema(store).types] == ["part_number"] + + def test_cancelled_reextraction_restarts_next_sync(self, isolated): + """An evolved schema whose pass is interrupted stays unapplied, so the + next sync completes it rather than leaving the new type unextracted.""" + store, services = isolated + _index_sources(store, 12) + services.provider.chat.return_value = SimpleNamespace(text=_INDUCED_JSON) + ensure_entities() + _index_sources(store, 8, start=12, text="part PX9001 near dock D-77") + services.provider.chat.return_value = SimpleNamespace(text=_REINDUCED_JSON) + cancelled = threading.Event() + cancelled.set() + ensure_entities(cancel=cancelled) + assert not _applied(store) + assert [t.name for t in load_schema(store).types] == ["part_number", "dock"] + + ensure_entities() # no re-induction needed; the pass just completes + assert _applied(store) + assert store.entity_value_counts("dock") == (8, 1) + + +class TestMergeSchemas: + def test_appends_only_unseen_extractors(self): + from lilbee.retrieval.entities import merge_schemas + + existing = _part_schema() + induced = EntitySchema( + types=[ + EntityType(name="renamed", kind=ExtractorKind.REGEX, pattern=r"PX\d{4}"), + EntityType(name="dock", kind=ExtractorKind.REGEX, pattern=r"D-\d{2}"), + ] + ) + merged = merge_schemas(existing, induced) + assert [t.name for t in merged.types] == ["part_number", "dock"] + + def test_name_collision_is_not_re_added(self): + """Same name, different pattern: keep the established one rather than + carry two types answering to the same noun.""" + from lilbee.retrieval.entities import merge_schemas + + induced = EntitySchema( + types=[EntityType(name="part_number", kind=ExtractorKind.REGEX, pattern=r"ZZ\d{4}")] + ) + merged = merge_schemas(_part_schema(), induced) + assert [(t.name, t.pattern) for t in merged.types] == [("part_number", r"PX\d{4}")] + + def test_nothing_new_returns_the_existing_schema(self): + from lilbee.retrieval.entities import merge_schemas + + existing = _part_schema() + assert merge_schemas(existing, _part_schema()) is existing + + def test_llm_kinds_dedupe_by_description(self): + from lilbee.retrieval.entities import merge_schemas + + existing = EntitySchema( + types=[EntityType(name="vessel", kind=ExtractorKind.LLM, description="Ships")] + ) + induced = EntitySchema( + types=[EntityType(name="boat", kind=ExtractorKind.LLM, description="ships")] + ) + assert merge_schemas(existing, induced) is existing + + +class TestCorpusDrifted: + def test_growth_below_the_factor_is_not_drift(self): + from lilbee.retrieval.entities.lifecycle import _corpus_drifted + + assert not _corpus_drifted(100, 149) + assert _corpus_drifted(100, 150) + + def test_small_corpora_drift_on_any_growth(self): + from lilbee.retrieval.entities.lifecycle import _corpus_drifted + + assert _corpus_drifted(2, 3) + assert not _corpus_drifted(2, 2) + + def test_shrinking_is_never_drift(self): + from lilbee.retrieval.entities.lifecycle import _corpus_drifted + + assert not _corpus_drifted(100, 40) + + class TestStatusEntities: def test_status_reports_types_and_rows(self, isolated): from lilbee.app.status import gather_status store, _services = isolated _index_chunks(store, ["part PX4471"]) - save_schema(_part_schema(), store) + save_schema(_part_schema(), store, applied=False, source_count=1) ensure_entities() result = gather_status() assert result.entities is not None diff --git a/tests/test_query.py b/tests/test_query.py index 7b41d8873..f38125b67 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1919,10 +1919,12 @@ def part_schema(self, mock_svc): EntityType(name="depot", kind=ExtractorKind.SPACY, pattern="GPE"), ] ) - mock_svc.store.entity_schema_state.return_value = ( - json.dumps(schema.model_dump(mode="json")), - True, - ) + mock_svc.store.entity_schema_state.return_value = { + "schema_json": json.dumps(schema.model_dump(mode="json")), + "applied": True, + "source_count": 1, + "updated_at": "2026-01-01T00:00:00+00:00", + } return schema def test_distinct_count_answers_exactly(self, mock_svc, part_schema): From 241b53f8cfb5e75f5914ff315fe6412d306223f0 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:12:13 -0400 Subject: [PATCH 74/77] server: reuse the mismatch-detail helper in the shared stream resolver The shared resolver inlined the same dims-match expression the helper already owned, and the chat stream's delegation left the helper with no callers. --- src/lilbee/server/handlers/rag.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lilbee/server/handlers/rag.py b/src/lilbee/server/handlers/rag.py index e8b85a9c4..7ddf67b6c 100644 --- a/src/lilbee/server/handlers/rag.py +++ b/src/lilbee/server/handlers/rag.py @@ -697,8 +697,11 @@ def _resolve_stream_context( ) except EmbeddingModelMismatchError as mismatch: # detail carries the index's embedder so the client can offer to adopt it. - detail = mismatch.persisted_model if mismatch.dims_match else None - frame = sse_error(str(mismatch), code=SseErrorCode.INDEX_EMBEDDER_MISMATCH, detail=detail) + frame = sse_error( + str(mismatch), + code=SseErrorCode.INDEX_EMBEDDER_MISMATCH, + detail=_mismatch_detail(mismatch), + ) return [], None, [frame] if rag is None: return [], None, [sse_error("No relevant documents found.")] From f0ab458a640115caea30a8c694d20304c59ba0d3 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:18:28 -0400 Subject: [PATCH 75/77] Drop a stray blank line the merge left in the chat screen --- src/lilbee/cli/tui/screens/chat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lilbee/cli/tui/screens/chat.py b/src/lilbee/cli/tui/screens/chat.py index 22395c271..656b18219 100644 --- a/src/lilbee/cli/tui/screens/chat.py +++ b/src/lilbee/cli/tui/screens/chat.py @@ -546,7 +546,6 @@ def _submit_draft(self, chat_input: ChatInput, value: str) -> None: return self._send_message(text) - def _ready_to_submit(self, text: str) -> bool: """Gate a submit: busy, consumed, empty, and keep-the-draft cases say no.""" if self._reject_submit_when_busy() or self._dismiss_overlay_on_submit() or not text: From a02fb97fc0e220f2f48ea1ec7c19729f0211aa94 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:22:10 -0400 Subject: [PATCH 76/77] tests: crawl integration test joins the worker it starts The test waited for the sync flag and returned while the crawl worker's tail (browser teardown, slow on Windows) was still running, tripping the leaked-worker guard whose short grace is meant for mocked no-op targets. A test that runs a real task-bar worker owns the thread; join it before returning. --- tests/integration/test_tui_integration.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/integration/test_tui_integration.py b/tests/integration/test_tui_integration.py index d8b5da0ad..f730cff65 100644 --- a/tests/integration/test_tui_integration.py +++ b/tests/integration/test_tui_integration.py @@ -10,6 +10,8 @@ from __future__ import annotations +import threading + import pytest from textual.app import ComposeResult from textual.widgets import DataTable, Footer @@ -21,6 +23,24 @@ pytestmark = pytest.mark.slow +# A test that runs a REAL task-bar worker (a crawl, a sync) owns that thread +# and must outwait its teardown; sized for Windows runners, where browser +# shutdown alone can take tens of seconds. +_TASK_WORKER_JOIN_S = 120.0 + + +def _join_task_workers() -> None: + """Join every live task-bar worker this test started. + + The UI signals (sync flag, task-bar state) flip before the worker + thread's tail finishes (crawler/browser teardown); returning while it + runs trips the conftest leak guard, whose short grace is meant for + mocked no-op targets, not a real crawl. + """ + for thread in threading.enumerate(): + if thread.name.startswith("task-"): + thread.join(timeout=_TASK_WORKER_JOIN_S) + class _IntegrationChatApp(LilbeeAppHost): """Minimal app that pushes ChatScreen with real services.""" @@ -315,6 +335,7 @@ async def test_crawl_becomes_searchable(self, rag_pipeline, monkeypatch) -> None results = get_services().searcher.search("bioluminescent jellyfish luciferin") assert len(results) > 0, "Expected crawled content to be searchable" finally: + _join_task_workers() server.clear() server.stop() From b4d0d219962af816d1832cc64682045a990606ee Mon Sep 17 00:00:00 2001 From: Tobias <5562156+tobocop2@users.noreply.github.com> Date: Sat, 18 Jul 2026 03:02:45 -0400 Subject: [PATCH 77/77] Index table structure and layout-ordered text from xberg extraction (#551) Tables in PDFs were only ever indexed as flattened text, so row/column meaning was lost to both BM25 and the embedder. With table_extraction on, xberg's table structure recognition runs during extraction and each table is indexed as its own chunk carrying the markdown serialization (new chunk_type 'table', covered by the 'raw' search scope), with long tables split header-repeating. With layout_detection on, page text is ordered by detected reading order and the running header/footer bands are stripped. Both flags are off by default and marked reindex-required; the xberg pin moves to 1.0.0rc29. bb-9t16 bb-orox --- docs/usage.md | 2 + pyproject.toml | 2 +- src/lilbee/app/settings_map.py | 15 + src/lilbee/core/config/model.py | 12 + src/lilbee/data/chunk.py | 24 +- src/lilbee/data/ingest/extract.py | 93 +++- src/lilbee/data/store/core.py | 6 +- src/lilbee/data/store/lance_helpers.py | 6 +- src/lilbee/data/store/types.py | 7 +- src/lilbee/retrieval/query/searcher.py | 10 +- tests/test_ingest.py | 192 ++++++++ tests/test_settings.py | 40 ++ tests/test_store.py | 13 + uv.lock | 628 ++++++++++++------------- 14 files changed, 715 insertions(+), 335 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 2ea68870b..768f22df2 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -762,6 +762,8 @@ effect on already-indexed material. | `LILBEE_CHUNK_OVERLAP` | `100` | Overlap tokens between adjacent chunks | | `LILBEE_MAX_EMBED_CHARS` | `2000` | Max characters per chunk passed to the embedder | | `LILBEE_SEMANTIC_CHUNKING` | `false` | Experimental topic-aware chunking. See [Semantic chunking](#semantic-chunking) | +| `LILBEE_TABLE_EXTRACTION` | `false` | Recognize table structure in PDFs and index each table as its own chunk, with long tables split so the header row repeats | +| `LILBEE_LAYOUT_DETECTION` | `false` | Layout-aware PDF extraction: text follows the detected reading order and running headers/footers are stripped | | `LILBEE_TOPIC_THRESHOLD` | `0.75` | Cosine boundary threshold for semantic chunking (lower = more splits) | | `LILBEE_EMBEDDING_DIM` | `768` | Embedding dimensionality. Must match the embedding model | | `LILBEE_EMBED_REPLICAS` | `1` | Embedding servers to run in parallel, one per spare GPU, for large-scale ingest. Capped at runtime by the GPUs with room after the chat model is placed | diff --git a/pyproject.toml b/pyproject.toml index 6f2a16fb7..1a5383dbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ classifiers = [ dependencies = [ "lancedb", - "xberg>=1.0.0rc25", + "xberg>=1.0.0rc29", "filelock", "tree-sitter-language-pack>=1.8.0,<2.0", "typer>=0.12", diff --git a/src/lilbee/app/settings_map.py b/src/lilbee/app/settings_map.py index ad897f9f5..82275ef6a 100644 --- a/src/lilbee/app/settings_map.py +++ b/src/lilbee/app/settings_map.py @@ -167,6 +167,21 @@ def get_default(key: str) -> object: group=SettingGroup.INGEST, help_text="Size chunks by real embedder tokens, not chars (changes invalidate the index)", ), + "table_extraction": SettingDef( + bool, + nullable=False, + group=SettingGroup.INGEST, + help_text="Index each extracted table as its own chunk (changes invalidate the index)", + ), + "layout_detection": SettingDef( + bool, + nullable=False, + group=SettingGroup.INGEST, + help_text=( + "Layout-aware PDF extraction: reading order plus header/footer " + "stripping (changes invalidate the index)" + ), + ), "embedding_model": SettingDef( str, nullable=False, diff --git a/src/lilbee/core/config/model.py b/src/lilbee/core/config/model.py index 1569540e8..6d6da193f 100644 --- a/src/lilbee/core/config/model.py +++ b/src/lilbee/core/config/model.py @@ -154,6 +154,18 @@ class Config(BaseSettings): # chunks, so a library must be reindexed. Applies to the plain and heading # chunkers; the semantic chunker sizes by characters and ignores it. token_sizing: bool = ConfigField(default=False, writable=True, reindex=True) + # Index each extracted table as its own chunk: xberg recognizes table + # structure during extraction and the markdown serialization is embedded + # alongside the prose chunks, so tabular data is retrievable as a unit. + # Off by default: recognition costs extraction time, and toggling changes + # what gets indexed, so a library must be reindexed. + table_extraction: bool = ConfigField(default=False, writable=True, reindex=True) + # Layout-aware PDF extraction: xberg's layout detection orders page text + # by detected reading order (multi-column PDFs stop interleaving) and the + # running header/footer bands are stripped. Off by default: detection runs + # an extra model pass per page, and toggling changes extracted text, so a + # library must be reindexed. + layout_detection: bool = ConfigField(default=False, writable=True, reindex=True) server_host: str = "127.0.0.1" server_port: int = Field(default=0, ge=0, le=65535) cors_origins: list[str] = Field(default_factory=list) diff --git a/src/lilbee/data/chunk.py b/src/lilbee/data/chunk.py index b1cfccf3e..b5a36251d 100644 --- a/src/lilbee/data/chunk.py +++ b/src/lilbee/data/chunk.py @@ -7,7 +7,7 @@ from lilbee.core.config import active_config if TYPE_CHECKING: - from xberg import ChunkingConfig, ChunkSizing, EmbeddingConfig + from xberg import ChunkingConfig, ChunkSizing, EmbeddingConfig, TableChunkingMode CHARS_PER_TOKEN = 4 @@ -61,6 +61,20 @@ def _semantic_embedding_config() -> EmbeddingConfig: return EmbeddingConfig(model=model) +def _table_chunking() -> TableChunkingMode | None: + """Header-repeating table splits when table extraction is on, else xberg's default. + + REPEAT_HEADER carries the header row into every piece of a long table, so + no chunk holds headerless rows. + """ + config = active_config() + if not config.table_extraction: + return None + from xberg import TableChunkingMode + + return TableChunkingMode.REPEAT_HEADER + + def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: """Build an xberg ChunkingConfig from the current cfg.""" from xberg import ChunkingConfig @@ -76,9 +90,15 @@ def build_chunking_config(*, use_semantic: bool = True) -> ChunkingConfig: topic_threshold=config.topic_threshold, max_characters=max_chars, overlap=max_overlap, + table_chunking=_table_chunking(), ) max_size, max_overlap, sizing = _size_params() - return ChunkingConfig(max_characters=max_size, overlap=max_overlap, sizing=sizing) + return ChunkingConfig( + max_characters=max_size, + overlap=max_overlap, + sizing=sizing, + table_chunking=_table_chunking(), + ) def chunk_text( diff --git a/src/lilbee/data/ingest/extract.py b/src/lilbee/data/ingest/extract.py index 9651346d9..70bcb2c34 100644 --- a/src/lilbee/data/ingest/extract.py +++ b/src/lilbee/data/ingest/extract.py @@ -34,7 +34,7 @@ ) if TYPE_CHECKING: - from xberg import ExtractedDocument, ExtractionConfig, OcrConfig + from xberg import ExtractedDocument, ExtractionConfig, OcrConfig, PdfConfig, Table log = logging.getLogger(__name__) @@ -133,6 +133,44 @@ def _ocr_force_requested() -> bool: return os.environ.get("LILBEE_OCR_FORCE", "").strip().lower() in {"1", "true", "yes"} +# Fraction of page height treated as the running header/footer band when +# layout detection is on. Conservative: strips only the outermost 5%. +_TOP_MARGIN_FRACTION = 0.05 +_BOTTOM_MARGIN_FRACTION = 0.05 + + +def _pdf_options() -> PdfConfig | None: + """PdfConfig for the enabled opt-in ingest features, or None when all are off. + + Table extraction turns on structure recognition; layout detection orders + text by detected reading order and strips the header/footer margin bands. + """ + config = active_config() + if not (config.table_extraction or config.layout_detection): + return None + from xberg import PdfConfig + + kwargs: dict[str, Any] = {} + if config.table_extraction: + kwargs["extract_tables"] = True + if config.layout_detection: + kwargs.update( + reading_order=True, + top_margin_fraction=_TOP_MARGIN_FRACTION, + bottom_margin_fraction=_BOTTOM_MARGIN_FRACTION, + ) + return PdfConfig(**kwargs) + + +def _layout_options() -> dict[str, Any]: + """ExtractionConfig kwargs enabling layout detection, empty when off.""" + if not active_config().layout_detection: + return {} + from xberg import LayoutDetectionConfig + + return {"layout": LayoutDetectionConfig(), "use_layout_for_markdown": True} + + def extraction_config(mode: ExtractMode, *, ocr_token: str | None = None) -> ExtractionConfig: """Build ExtractionConfig for the given extraction mode.""" from xberg import ExtractionConfig, PageConfig @@ -152,6 +190,8 @@ def extraction_config(mode: ExtractMode, *, ocr_token: str | None = None) -> Ext pages=PageConfig(extract_pages=True, insert_page_markers=False), ocr=ocr, force_ocr=force_ocr, + pdf_options=_pdf_options(), + **_layout_options(), ) return ExtractionConfig( chunking=chunking, @@ -229,6 +269,20 @@ def _capture_result_page_texts( page_texts_out.append(_page_text_record(source_name, 0, doc.content, content_type)) +def _document_tables(doc: ExtractedDocument) -> list[Table]: + """The result tables to index as dedicated chunks, when table extraction is on. + + Each table becomes its own chunk carrying xberg's markdown serialization and + page metadata. The table's flattened text also stays inside the content + chunks: stripping it there would tear holes in reading-order prose and the + page-text export, so both are indexed deliberately. The dedicated table + chunk adds the structured serialization for targeted retrieval. + """ + if not active_config().table_extraction: + return [] + return [t for t in (doc.tables or []) if t.markdown and t.markdown.strip()] + + def _warn_empty_ocr(source_name: str, media: str) -> None: """Warn that extraction yielded no text and point to the vision-model remedy.""" log.warning( @@ -293,7 +347,8 @@ def _tick() -> None: ) ) - if not doc.chunks: + tables = _document_tables(doc) + if not doc.chunks and not tables: if content_type in (PDF_CONTENT_TYPE, IMAGE_CONTENT_TYPE): _warn_empty_ocr(source_name, "scanned documents") return [] @@ -309,11 +364,17 @@ def _tick() -> None: ExtractEvent(file=source_name, page=page_count, total_pages=page_count), ) - texts = [chunk.content for chunk in doc.chunks] + # Content chunks and table serializations share one embed batch; the vector + # list is split back apart below by position. + texts = [chunk.content for chunk in doc.chunks or []] + table_texts = [table.markdown for table in tables] vectors = await to_ingest_thread( - get_services().embedder.embed_batch, texts, source=source_name, on_progress=on_progress + get_services().embedder.embed_batch, + texts + table_texts, + source=source_name, + on_progress=on_progress, ) - return [ + records = [ ChunkRecord( source=source_name, content_type=content_type, @@ -326,8 +387,28 @@ def _tick() -> None: chunk_index=chunk.metadata.chunk_index, vector=vec, ) - for chunk, text, vec in zip(doc.chunks, texts, vectors, strict=True) + for chunk, text, vec in zip(doc.chunks or [], texts, vectors[: len(texts)], strict=True) ] + # Table chunk indices continue after the content chunks so a source's + # (source, chunk_index) pairs stay unique. + records.extend( + ChunkRecord( + source=source_name, + content_type=content_type, + chunk_type=ChunkType.TABLE, + page_start=table.page_number, + page_end=table.page_number, + line_start=0, + line_end=0, + chunk=text, + chunk_index=len(texts) + i, + vector=vec, + ) + for i, (table, text, vec) in enumerate( + zip(tables, table_texts, vectors[len(texts) :], strict=True) + ) + ) + return records async def ingest_markdown( diff --git a/src/lilbee/data/store/core.py b/src/lilbee/data/store/core.py index 3bf452922..0dcea5566 100644 --- a/src/lilbee/data/store/core.py +++ b/src/lilbee/data/store/core.py @@ -487,7 +487,8 @@ def bm25_probe( ) -> list[SearchChunk]: """Quick BM25-only search for confidence checking. Returns up to top_k results. - When *chunk_type* is set, only chunks of that type ("raw" or "wiki") are returned. + When *chunk_type* is set, only matching chunks are returned; a "raw" + filter also covers table chunks. """ table = self.open_table(CHUNKS_TABLE) if table is None: @@ -522,7 +523,8 @@ def search( Results with distance > max_distance are filtered out (vector-only path). Pass max_distance=0 to disable filtering. - When *chunk_type* is set, only chunks of that type ("raw" or "wiki") are returned. + When *chunk_type* is set, only matching chunks are returned; a "raw" + filter also covers table chunks. Raises ``EmbeddingModelMismatchError`` if the persisted ``_meta`` row was written under a different embedding model than the current ``cfg``. diff --git a/src/lilbee/data/store/lance_helpers.py b/src/lilbee/data/store/lance_helpers.py index 6565f3ac1..4ed67bb4f 100644 --- a/src/lilbee/data/store/lance_helpers.py +++ b/src/lilbee/data/store/lance_helpers.py @@ -120,11 +120,13 @@ def _chunk_type_predicate(chunk_type: ChunkType | str) -> str: Rows written before ``chunk_type`` was populated land as NULL. They are semantically raw, so a ``'raw'`` filter still includes them; a - ``'wiki'`` filter excludes them. + ``'wiki'`` filter excludes them. Table chunks are document content, + so the ``'raw'`` scope covers them too; filter on ``'table'`` to + isolate them. """ escaped = escape_sql_string(chunk_type) if chunk_type == ChunkType.RAW: - return f"(chunk_type = '{escaped}' OR chunk_type IS NULL)" + return f"(chunk_type IN ('{escaped}', '{ChunkType.TABLE}') OR chunk_type IS NULL)" return f"chunk_type = '{escaped}'" diff --git a/src/lilbee/data/store/types.py b/src/lilbee/data/store/types.py index 6ff0829c8..90bf20bcf 100644 --- a/src/lilbee/data/store/types.py +++ b/src/lilbee/data/store/types.py @@ -67,11 +67,14 @@ class ChunkWrite(NamedTuple): class ChunkType(StrEnum): """Values for the ``chunk_type`` column. - Everything ingests as ``RAW`` except wiki pages written by the wiki - producer; callers filter with ``Store.search(chunk_type=...)``. + Documents ingest as ``RAW``, extracted tables as ``TABLE`` (when table + extraction is on), and wiki pages written by the wiki producer as + ``WIKI``. Callers filter with ``Store.search(chunk_type=...)``; a ``RAW`` + filter also covers table chunks, since both are document content. """ RAW = "raw" + TABLE = "table" WIKI = "wiki" diff --git a/src/lilbee/retrieval/query/searcher.py b/src/lilbee/retrieval/query/searcher.py index a1f7c7cf5..e6ceda67a 100644 --- a/src/lilbee/retrieval/query/searcher.py +++ b/src/lilbee/retrieval/query/searcher.py @@ -528,8 +528,9 @@ def search( """Embed question and search with expansion, HyDE, and concept boost. Returns up to top_k*2 candidates for downstream filtering. - When *chunk_type* is set (``"raw"`` or ``"wiki"``), only chunks of - that type are returned. An explicit ``chunk_type`` always wins + When *chunk_type* is set (``"raw"`` or ``"wiki"``), only matching + chunks are returned (``"raw"`` also covers table chunks). An + explicit ``chunk_type`` always wins over the ``wiki:``/``raw:`` prefix shortcut in *question* so the user-facing scope choice has the final say. @@ -682,8 +683,9 @@ def build_rag_context( ) -> tuple[list[SearchChunk], list[ChatMessage]] | None: """Build RAG context from search results. - ``chunk_type`` restricts the pool to ``"raw"`` or ``"wiki"`` rows; - ``None`` (default) searches the mixed pool. + ``chunk_type`` restricts the pool to ``"raw"`` (which covers table + chunks too) or ``"wiki"`` rows; ``None`` (default) searches the + mixed pool. """ retrieval_query = question if history and self._config.history_rewrite: diff --git a/tests/test_ingest.py b/tests/test_ingest.py index a2b369479..9e0cf663e 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -137,6 +137,7 @@ def _make_xberg_result( num_chunks=1, has_pages=False, document=None, + tables=None, ): """Build a mock xberg ExtractionResult.""" chunks = [] @@ -159,6 +160,7 @@ def _make_xberg_result( result.chunks = chunks result.content = text result.document = document + result.tables = tables if tables is not None else [] result.pages = ( [mock.MagicMock(page_number=i + 1, content=chunks[i].content) for i in range(num_chunks)] if has_pages @@ -167,12 +169,22 @@ def _make_xberg_result( return result +def _make_table(markdown="| h1 | h2 |\n|---|---|\n| a | b |", page_number=1): + """Build a mock xberg Table carrying its markdown serialization.""" + table = mock.MagicMock() + table.markdown = markdown + table.page_number = page_number + return table + + def _make_empty_result(): """Build a mock xberg ExtractionResult with no chunks.""" result = mock.MagicMock() result.chunks = [] result.content = "" result.document = None + result.tables = [] + result.pages = [] return result @@ -2087,6 +2099,186 @@ def test_topic_threshold_propagates_to_every_mode(self, monkeypatch): assert config["chunking"].chunker_type == "semantic" assert config["chunking"].topic_threshold == pytest.approx(0.42, abs=1e-5) + def test_table_extraction_off_sets_no_pdf_options(self): + """Default config leaves pdf_options unset: exact pre-feature behavior.""" + from lilbee.data.ingest import ExtractMode, extraction_config + + config = extraction_config(ExtractMode.PAGINATED) + assert config.get("pdf_options") is None + + def test_table_extraction_sets_pdf_options_and_table_chunking(self, monkeypatch): + """The flag turns on xberg table recognition and header-repeating splits.""" + from xberg import TableChunkingMode + + from lilbee.core.config import cfg + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.setattr(cfg, "table_extraction", True) + config = extraction_config(ExtractMode.PAGINATED) + assert config["pdf_options"].extract_tables is True + assert config["chunking"].table_chunking == TableChunkingMode.REPEAT_HEADER + + def test_table_extraction_markdown_mode_skips_pdf_options(self, monkeypatch): + """Non-paginated formats have no PdfConfig but still chunk tables with headers.""" + from xberg import TableChunkingMode + + from lilbee.core.config import cfg + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.setattr(cfg, "table_extraction", True) + config = extraction_config(ExtractMode.MARKDOWN) + assert config.get("pdf_options") is None + assert config["chunking"].table_chunking == TableChunkingMode.REPEAT_HEADER + + def test_layout_detection_off_sets_no_layout(self): + """Default config leaves layout detection unset: exact pre-feature behavior.""" + from lilbee.data.ingest import ExtractMode, extraction_config + + config = extraction_config(ExtractMode.PAGINATED) + assert config.get("layout") is None + assert config.get("pdf_options") is None + + def test_layout_detection_sets_layout_and_reading_order(self, monkeypatch): + """The flag enables layout detection, reading order, and margin stripping.""" + from lilbee.core.config import cfg + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.setattr(cfg, "layout_detection", True) + config = extraction_config(ExtractMode.PAGINATED) + assert config["layout"] is not None + assert config["use_layout_for_markdown"] is True + pdf = config["pdf_options"] + assert pdf.reading_order is True + assert pdf.top_margin_fraction == pytest.approx(0.05) + assert pdf.bottom_margin_fraction == pytest.approx(0.05) + # extract_tables stays at xberg's default (True); table INDEXING is + # gated separately by cfg.table_extraction in _document_tables. + assert pdf.extract_tables is True + + def test_layout_detection_markdown_mode_unchanged(self, monkeypatch): + """Layout detection is visual-format work; non-paginated formats skip it.""" + from lilbee.core.config import cfg + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.setattr(cfg, "layout_detection", True) + config = extraction_config(ExtractMode.MARKDOWN) + assert config.get("layout") is None + assert config.get("pdf_options") is None + + def test_layout_and_tables_combine_in_one_pdf_config(self, monkeypatch): + """Both flags land in the same PdfConfig.""" + from lilbee.core.config import cfg + from lilbee.data.ingest import ExtractMode, extraction_config + + monkeypatch.setattr(cfg, "table_extraction", True) + monkeypatch.setattr(cfg, "layout_detection", True) + config = extraction_config(ExtractMode.PAGINATED) + pdf = config["pdf_options"] + assert pdf.extract_tables is True + assert pdf.reading_order is True + + +class TestTableChunks: + """Table indexing in ingest_document, driven by cfg.table_extraction.""" + + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) + async def test_config_off_ignores_tables(self, mock_kf, isolated_env): + """With the flag off, result tables are not indexed: current behavior.""" + mock_kf.return_value = _make_xberg_result(tables=[_make_table()]) + from lilbee.data.ingest import ingest_document + from lilbee.data.store import ChunkType + + f = isolated_env / "test.pdf" + f.write_bytes(b"fake") + records = await ingest_document(f, "test.pdf", "pdf") + assert len(records) == 1 + assert all(r["chunk_type"] == ChunkType.RAW for r in records) + + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) + async def test_tables_indexed_as_table_chunks(self, mock_kf, isolated_env, monkeypatch): + """Each table becomes its own chunk: markdown text, page span, TABLE type.""" + from lilbee.core.config import cfg + + monkeypatch.setattr(cfg, "table_extraction", True) + tables = [ + _make_table(markdown="| a | b |\n|---|---|\n| 1 | 2 |", page_number=2), + _make_table(markdown="| x |\n|---|\n| y |", page_number=5), + ] + mock_kf.return_value = _make_xberg_result(tables=tables) + from lilbee.data.ingest import ingest_document + from lilbee.data.store import ChunkType + + f = isolated_env / "test.pdf" + f.write_bytes(b"fake") + records = await ingest_document(f, "test.pdf", "pdf") + assert len(records) == 3 + table_records = [r for r in records if r["chunk_type"] == ChunkType.TABLE] + assert len(table_records) == 2 + assert table_records[0]["chunk"] == tables[0].markdown + assert table_records[0]["page_start"] == 2 + assert table_records[0]["page_end"] == 2 + assert table_records[1]["page_start"] == 5 + # Indices continue after the content chunks so (source, chunk_index) stays unique. + assert [r["chunk_index"] for r in records] == [0, 1, 2] + assert all(r["vector"] for r in table_records) + assert all(r["content_type"] == "pdf" for r in table_records) + + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) + async def test_spanning_cell_markdown_passes_through_verbatim( + self, mock_kf, isolated_env, monkeypatch + ): + """xberg's serialization of spanning cells is indexed exactly as produced.""" + from lilbee.core.config import cfg + + monkeypatch.setattr(cfg, "table_extraction", True) + spanned = "| h1 | h1 | h2 |\n|---|---|---|\n| merged | merged | v |" + mock_kf.return_value = _make_xberg_result(tables=[_make_table(markdown=spanned)]) + from lilbee.data.ingest import ingest_document + from lilbee.data.store import ChunkType + + f = isolated_env / "test.pdf" + f.write_bytes(b"fake") + records = await ingest_document(f, "test.pdf", "pdf") + assert records[-1]["chunk_type"] == ChunkType.TABLE + assert records[-1]["chunk"] == spanned + + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) + async def test_blank_table_markdown_skipped(self, mock_kf, isolated_env, monkeypatch): + """A table with no usable serialization contributes no chunk.""" + from lilbee.core.config import cfg + + monkeypatch.setattr(cfg, "table_extraction", True) + mock_kf.return_value = _make_xberg_result(tables=[_make_table(markdown=" ")]) + from lilbee.data.ingest import ingest_document + from lilbee.data.store import ChunkType + + f = isolated_env / "test.pdf" + f.write_bytes(b"fake") + records = await ingest_document(f, "test.pdf", "pdf") + assert len(records) == 1 + assert records[0]["chunk_type"] == ChunkType.RAW + + @mock.patch("lilbee.data.xberg_extract.aextract_document", new_callable=mock.AsyncMock) + async def test_tables_without_text_chunks_still_indexed( + self, mock_kf, isolated_env, monkeypatch + ): + """A document whose only content is tables is not dropped as empty.""" + from lilbee.core.config import cfg + + monkeypatch.setattr(cfg, "table_extraction", True) + doc = _make_empty_result() + doc.tables = [_make_table(page_number=1)] + mock_kf.return_value = doc + from lilbee.data.ingest import ingest_document + from lilbee.data.store import ChunkType + + f = isolated_env / "test.pdf" + f.write_bytes(b"fake") + records = await ingest_document(f, "test.pdf", "pdf") + assert len(records) == 1 + assert records[0]["chunk_type"] == ChunkType.TABLE + assert records[0]["chunk_index"] == 0 + class TestClassifyXbergParityFormats: """Formats xberg extracts that the map must not silently drop.""" diff --git a/tests/test_settings.py b/tests/test_settings.py index 30f915e51..e1b8dd71d 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -220,6 +220,46 @@ def test_replicas_reject_negative(self): Config(embed_replicas=-1) +class TestTableExtractionSetting: + """The table-extraction flag is writable, grouped with ingest, and reindex-marked.""" + + def test_table_extraction_in_settings_map(self): + from lilbee.app.settings_map import SETTINGS_MAP, get_default + + defn = SETTINGS_MAP["table_extraction"] + assert defn.writable is True + assert defn.nullable is False + assert defn.type is bool + assert defn.group == "Ingest" + assert get_default("table_extraction") is False + + def test_table_extraction_requires_reindex(self): + from lilbee.config_meta import REINDEX_FIELDS, WRITABLE_CONFIG_FIELDS + + assert "table_extraction" in WRITABLE_CONFIG_FIELDS + assert "table_extraction" in REINDEX_FIELDS + + +class TestLayoutDetectionSetting: + """The layout-detection flag is writable, grouped with ingest, and reindex-marked.""" + + def test_layout_detection_in_settings_map(self): + from lilbee.app.settings_map import SETTINGS_MAP, get_default + + defn = SETTINGS_MAP["layout_detection"] + assert defn.writable is True + assert defn.nullable is False + assert defn.type is bool + assert defn.group == "Ingest" + assert get_default("layout_detection") is False + + def test_layout_detection_requires_reindex(self): + from lilbee.config_meta import REINDEX_FIELDS, WRITABLE_CONFIG_FIELDS + + assert "layout_detection" in WRITABLE_CONFIG_FIELDS + assert "layout_detection" in REINDEX_FIELDS + + class TestMemoryTuningSettingsMap: """The dynamic-ctx tuning knobs are surfaced in the TUI settings map.""" diff --git a/tests/test_store.py b/tests/test_store.py index 73514aa36..5e27a00dc 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1686,6 +1686,19 @@ def test_wiki_does_not_match_null(self): assert "IS NULL" not in pred assert pred == "chunk_type = 'wiki'" + def test_raw_scope_includes_table_chunks(self): + """Table chunks are document content, so the raw scope covers them.""" + from lilbee.data.store.lance_helpers import _chunk_type_predicate + + pred = _chunk_type_predicate("raw") + assert "'table'" in pred + + def test_table_filter_is_exact(self): + from lilbee.data.store.lance_helpers import _chunk_type_predicate + + pred = _chunk_type_predicate("table") + assert pred == "chunk_type = 'table'" + class TestEmbeddingModelGate: """Refuse search/ingest when cfg.embedding_model drifts from the persisted _meta row.""" diff --git a/uv.lock b/uv.lock index a1db39f00..0bd43067f 100644 --- a/uv.lock +++ b/uv.lock @@ -12,9 +12,6 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'win32'", ] -[options] -prerelease-mode = "allow" - [[package]] name = "aiofiles" version = "25.1.0" @@ -665,86 +662,86 @@ wheels = [ [[package]] name = "coverage" -version = "7.15.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4a/83/6051c2a2feab48ae5bd27c84ef047191d2d4a3172f689e38eaa48ed17db1/coverage-7.15.1.tar.gz", hash = "sha256:165e9949eaf222ef1f018635d0d7f368a23bfe0212af558534c40d8c04686d67", size = 927640 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/ee/5095e07a0986930174fc153fd0bb115f7f2724f5344cc672e016d4f23a4e/coverage-7.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6506330b4a8dcf53b95bd84d8d0e817107cdb3fc1438e835029cdf0bc6612eb0", size = 221320 }, - { url = "https://files.pythonhosted.org/packages/43/7a/7e2598ce98433a214020a61c0755d5961f9f26df0c630dc73e06b86d3416/coverage-7.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8a51a8ec382c39d939cba0ab07ae949077ae4e842343bd4eed22d432358cea9", size = 221823 }, - { url = "https://files.pythonhosted.org/packages/8b/4f/67dac6a69139b490a239d91b6d5ef127982ffb89159769241656ab879ee5/coverage-7.15.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:18ea20e3922d7f8ca9e0ef1084408d08c4ad62d5e531cb9c1f6896a99297ebea", size = 252242 }, - { url = "https://files.pythonhosted.org/packages/05/37/5e439725a96de592309725550c8df639c359c209dcb4b152ed46032d4c0d/coverage-7.15.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:aab9902a64b8390e3b56e539fddae1d79a267807fe5cb0c18d7d2f544ce867e2", size = 254153 }, - { url = "https://files.pythonhosted.org/packages/0f/1c/671ba9e15f9651a99962fb588a1fe4fb267cae4bb85f9bb4ac4413c6f7ef/coverage-7.15.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cc316264317b07a9e90d7f2b4188a15e36e9b54e651081b791b0515fa612a29", size = 256261 }, - { url = "https://files.pythonhosted.org/packages/d6/3c/88305322b4d015b4536b7174b8f9b87f850f01af9d89008cdbf984b65ff2/coverage-7.15.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d4e47e7eea81a8ccf060a07627654151d929da62c7b715738387c200905cec89", size = 258222 }, - { url = "https://files.pythonhosted.org/packages/8d/20/d02e42333a57f266ded61ed4fb3821da1f2dc143734aaa3dcc9c117204c5/coverage-7.15.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c6829d9a3b55ad2b73ef5fda8302e5be03683789e88b1a079dcf4a773229c21d", size = 252368 }, - { url = "https://files.pythonhosted.org/packages/f3/d0/94ebc010926a191d83e83b1f1236f8bd0d14d0eb98b5c324aa4be2589a8e/coverage-7.15.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:764e045811f9c8cda436641f3f088283351d331a519b5807f19041cd0a68da1c", size = 253953 }, - { url = "https://files.pythonhosted.org/packages/2e/a6/2c8553191da0c2da2c43ff294fb9bab57a5b0fae03560304a82a8b5addc3/coverage-7.15.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:09b3b088aa24489c4082bcc35fcc8224281ab94a653dfb6d3f0c8165b0d628ab", size = 252016 }, - { url = "https://files.pythonhosted.org/packages/17/99/20f47986d8360fa1a31d9858dd2ba0be009d44ecfa8d02120b1eb93c028b/coverage-7.15.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7911b02f57053adf8164ae63edb1c26574d24dfccabadc5268cf69310a69a358", size = 255783 }, - { url = "https://files.pythonhosted.org/packages/17/36/fb61983b5259f78fa5e62ee74e91fe4de4c556fcbc9a3b9c9021c2f8b57b/coverage-7.15.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:664e279ed40599b8ed16f4db18d92a7e212c73129672bec8f5d96d4da48d2404", size = 251735 }, - { url = "https://files.pythonhosted.org/packages/6c/2e/d109d8d2d6c726ba2e78125297073918a2a91464f0ea777e5398fa3cf75c/coverage-7.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34fe7cf79d5f1f87f2e8ce7dc1c32950841f50e10d0120f263856acfad66de34", size = 252645 }, - { url = "https://files.pythonhosted.org/packages/bc/a0/5b3219cbb46135e14673427f2bf5890c4da4e6f655243692f3448aa3f42c/coverage-7.15.1-cp311-cp311-win32.whl", hash = "sha256:5e2d2536d2f57a354aa382ed303ac0e2e5c9522a508c05b998d26181b94163a7", size = 223414 }, - { url = "https://files.pythonhosted.org/packages/5b/80/24e13ade0b6df8090e2492f9c55044bf1423f7ef28c76fc1af8c827a61cb/coverage-7.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:c337da8fca7ea93ab43f3868cfcde6cf6dad32c3906b273cfbad5d7390bc423b", size = 223888 }, - { url = "https://files.pythonhosted.org/packages/2d/9e/34659930ae380491af48e2f5f5f9a2e99cd9fb7de3d30f27be5ac5425137/coverage-7.15.1-cp311-cp311-win_arm64.whl", hash = "sha256:db3403fdb7a94d5eb73e099befad8104d2a7d110a0f0d99df0de61c5d1fa756c", size = 223435 }, - { url = "https://files.pythonhosted.org/packages/d9/76/32c1826309beaf4604c54accef108fdd611e5e5e93f2f5192f050cd5f6bd/coverage-7.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d9476292594309db922cc841dd13b303b3c388f4c25d279884f7e2341c681f80", size = 221497 }, - { url = "https://files.pythonhosted.org/packages/db/5c/b88ce0d68fa550c7f3b58617fbf363bce64df5bf8295a01b627e4696e022/coverage-7.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c579056b0de461b3a62318b63d0b6ce90aed7f8158d3f00da094df82f29d189", size = 221854 }, - { url = "https://files.pythonhosted.org/packages/0e/fe/8509fd2a66fc4e0a829f76a0f0b1dc3cc163368352435b5f243168658077/coverage-7.15.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:23214bdbe226f2b0e9c66a7d6a1d59d4a88045dcf86e702cf0fe0d0935e3d615", size = 253359 }, - { url = "https://files.pythonhosted.org/packages/e5/81/c7009ed7ea9765adb2b9d095054d748266fae5f07ac6c5f925f33715fcde/coverage-7.15.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:df164be93b46b4825cc39339440a05edc54c4d1d865ba4a60fd43d151a2a1cd3", size = 256096 }, - { url = "https://files.pythonhosted.org/packages/21/52/dc8ee03968a5ba86e2da5aa48ddc9e3747bd65d63825fdce2d96acb9c5ff/coverage-7.15.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a524fca1a6f08927d9dc2d4c873cfb7bd7202c247f08b14bdc02424071b8b304", size = 257211 }, - { url = "https://files.pythonhosted.org/packages/b8/27/95d7623908da8937deb53d48efcdbf423907a47540e63c62fa21372c652b/coverage-7.15.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d70f3542cd38de85a9e257dcb1ac4c1ab4b6d7d2c2a645809207556628755d1c", size = 259473 }, - { url = "https://files.pythonhosted.org/packages/f1/3b/730d761928de97d585465680b568ae69622fb40716babadeabffe75cb51b/coverage-7.15.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d78aa537237212c4313aabe5e964b66acc86350ed19ebc56a3e202df33b6077b", size = 253759 }, - { url = "https://files.pythonhosted.org/packages/f1/fc/6b9277acff1f9484b6c12857af5774689d1a6a95e13265f7405329d2f5da/coverage-7.15.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a318112bb4f79d9d04766196d5a3388caa825908a6a9b052aa87de3d9aea7c61", size = 255131 }, - { url = "https://files.pythonhosted.org/packages/3d/f2/c704f86129594ba34e25a64695d2068c71d51c2b98907184d716c94f4aec/coverage-7.15.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e55d24cada901963eed5bc89fa562aa033f0d84b9d3de4ecf363737c13aed11e", size = 253275 }, - { url = "https://files.pythonhosted.org/packages/f6/29/80fee8af47de4a6dce71ccf2938491f444687a756af258a56d8469b8f1b0/coverage-7.15.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3c78f0cea7275342cf2adc2ad5fdd0aafa106ad91e66d573568f2fcf62c41df5", size = 257345 }, - { url = "https://files.pythonhosted.org/packages/20/21/a1e7d7ed1b48a8adf8fd5154d9e83fcc5ad8e6ff20ae00e44865057dce8d/coverage-7.15.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:86bd37eabe39977216f630a7fc1b698e7f5e81a191c7186013245c6c3d313f9d", size = 252844 }, - { url = "https://files.pythonhosted.org/packages/a7/8c/a4bc26e6ee207d412f3678f04d74be1550e83140563ca0e4997510579712/coverage-7.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c6db15c217693bdc3ca0b84de1ba9afafe1c14c26a8a29d77f4ed0de2b6132e2", size = 254716 }, - { url = "https://files.pythonhosted.org/packages/11/9d/8ad0266ecfada6353cf6627a1a02294cf55a907521b6ee0bd7b770cfd659/coverage-7.15.1-cp312-cp312-win32.whl", hash = "sha256:359f3fbe09a51500c51966596ee4ee4070b356552c70b3b2420eb200d68e0f76", size = 223554 }, - { url = "https://files.pythonhosted.org/packages/81/6d/24224929e06c6e05a93f738bc5f9e8e6ab658f8f1d9b823e7b85430e28b8/coverage-7.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:fa75dc099c126e941a9c0baa8ebd2cbc78bd778687534fe410baf754f6d9e374", size = 224087 }, - { url = "https://files.pythonhosted.org/packages/35/23/f81441dd01de88e53c97842e706907b307d9078918c3f4998b11e9ac7250/coverage-7.15.1-cp312-cp312-win_arm64.whl", hash = "sha256:26f89cf6d0634375f454fa71057945ad18edb0f1607a90fecf22c57dc3dc289a", size = 223472 }, - { url = "https://files.pythonhosted.org/packages/ca/1e/6fa289d7993a2a39f1b283ddb58c4bfec80f7800be654b8ba8a9f6a07c63/coverage-7.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:71ac4ca1658ca99160fd58cc6967110e989c34b04627f24ed6ec9f70fb24571a", size = 221519 }, - { url = "https://files.pythonhosted.org/packages/6f/e1/0db4902a0588234a70ab0218073c0b20fbc5c740aa35f91d360160a2ebc9/coverage-7.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:26a40cbf2b13bd94af53ee02a424cb3bb96a9edfac0d00834bd068512a62714b", size = 221895 }, - { url = "https://files.pythonhosted.org/packages/b4/cb/3719783865092dac5e08df842730305ee9ab1973ae7ddb6fbdf27d401f30/coverage-7.15.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4c5a5eff4ad4f9f7088fd3fc7a66d98d06566ee294b3b053309fb0a3b45be1e", size = 252882 }, - { url = "https://files.pythonhosted.org/packages/1c/5e/caf3abbdbb22629626160ffc9c017eb995b7cb11c0be46b974834cef1792/coverage-7.15.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:962aa56c1c9b016d681265880eb6acc9966029d2c4c559319cc43a1abbb9b59a", size = 255479 }, - { url = "https://files.pythonhosted.org/packages/d6/f1/d60f375bfe095fef944f0f19427aefdbf9bdd5a9571c41a4bf6e2f5fdb81/coverage-7.15.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1678eb2dc57a8ce67601b029582ef6d41e9e6ca22692aaeccd4107e40f27386c", size = 256715 }, - { url = "https://files.pythonhosted.org/packages/d7/17/8b0cbc90d02dc5adad4d9034c1824ec3fa567771b4c39d9c1e3f9b1431b8/coverage-7.15.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1174900a43f6f8c425fee10d7dbddc308adefcdc78aaced32357f5ab750a0e90", size = 258845 }, - { url = "https://files.pythonhosted.org/packages/92/29/c5e69f5fb75c322e9a3e4ef64d02eebfc3d66efceccc8514ff80a3c13a56/coverage-7.15.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:98847557a6859cadf693792ce89f440cb89692993f60dc6d3a7e35f3d340216f", size = 253098 }, - { url = "https://files.pythonhosted.org/packages/64/57/21144252fdd0c01d707d48fbcea13a80b0b7c42ced3f299f885ab8978c3a/coverage-7.15.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8697b2edb57143546a24389efc11e1b000cd5800fc20d84f04edb601e4a7cfb8", size = 254844 }, - { url = "https://files.pythonhosted.org/packages/59/2a/499a28a322b0ce6768328e6c5bb2e2ad00ac068a7c7adb2ecd8533c8c5d9/coverage-7.15.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6827ac0519be3fe91bf96b4060eb00d1d24f82649b29862cd75a3cfca248b02a", size = 252807 }, - { url = "https://files.pythonhosted.org/packages/2c/6f/928a95da5da8b60f2b00e1482c7787b3316188e6d2d227fb8e124ada43a1/coverage-7.15.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2de8ecbbc77c7e4d22572779920ed8979c69168675e96be3a548c996568c6c31", size = 256965 }, - { url = "https://files.pythonhosted.org/packages/16/10/889adbc1b8c9f866ed51e18a98bcafc0259fb9d29b81f50a719407c64ea8/coverage-7.15.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2b25f0f0fa5260df9d7bb55d47c8bdc23fa3382c1a18f7c9cae122e6c320b1ad", size = 252628 }, - { url = "https://files.pythonhosted.org/packages/1a/30/a5e1871e5d93416511f8e359d1ccebfe0cbb050a1bbf7dd20228533ec0cf/coverage-7.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a2effcbd93ae340a58db718fe4181d967f84d352c4cefeaab4ff82ce813901a", size = 254399 }, - { url = "https://files.pythonhosted.org/packages/2d/26/c36fbffd549dadbdd1a75827528fb00a4c46aa3187b007b750b1e2cebbf2/coverage-7.15.1-cp313-cp313-win32.whl", hash = "sha256:895e65c96aef0cecea250f6e35e9a32f11375514e1a0cb5210e0fda128c04e8e", size = 223564 }, - { url = "https://files.pythonhosted.org/packages/16/fc/becbb9d2c4206d242b9b1e1e8e24a42f7926c0200dd3c788b9fab4bb96d5/coverage-7.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6d0a28b63a0d75f9ed5118105d1154fc3aa40a8605a30d5d87e3d043ad90fe7", size = 224106 }, - { url = "https://files.pythonhosted.org/packages/d3/30/1cfc641461369b6858799fca61c0a8b5edc490c519bf7c636ffa6bbf556f/coverage-7.15.1-cp313-cp313-win_arm64.whl", hash = "sha256:b4ee9818e8bae3544379ad2c09b851c4fb886aaa8860d57a1c1316ddcb16db49", size = 223497 }, - { url = "https://files.pythonhosted.org/packages/f0/46/81961952e7aebfb38ad0ae4264e8954cc607a7af9e7ac111f9fa986595cc/coverage-7.15.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a886af95f59edf67d5770fd3564d53f4a8af93f25f8c1d60d27e00d7f5674ee8", size = 221560 }, - { url = "https://files.pythonhosted.org/packages/13/d2/ee14d715889f216baf47301d9f469e08fff6995552aaf67e897b282865f6/coverage-7.15.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:985657ebd707941de90d488d1cbb5efac20bdf81f7b91eba771624ccda4d36f4", size = 221894 }, - { url = "https://files.pythonhosted.org/packages/f3/38/f830bc6e6c2c5f23f43847125e6c650d378872f7eeba8d49f1d42193e8a9/coverage-7.15.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5bbe2a06e0a5e1404d9ffbdb49b819bbd6a3bb198ebea4c8dfe7ad9f1e1c2e81", size = 252938 }, - { url = "https://files.pythonhosted.org/packages/e1/53/0d3dd963631259d794c898735d5436e68d6a8d40749c419a07ff7c171469/coverage-7.15.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bde0fe24083d0b7b3dbafa7a09f0796410af1afa2523f28f5f208d8340a4aaca", size = 255445 }, - { url = "https://files.pythonhosted.org/packages/b1/fd/aabed228557565c958259251b89bab8c5669b31291fa63b3e2154ebb017a/coverage-7.15.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f89f7453d6d46db14cf233e2cd8edcd78de2b9c49d4f1dc109590b4e5dbfbb74", size = 256790 }, - { url = "https://files.pythonhosted.org/packages/bc/aa/1cc888e5d3623e603c4e5399653cb25728bb2b40d7519188a3e293d24620/coverage-7.15.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc3656c9ecc27b36bd0907455b77f83c0069ca9ad4a66dec892b76c696eb6047", size = 259104 }, - { url = "https://files.pythonhosted.org/packages/5f/61/fc16d5f5e53098dae41efa21e8ccc611a9b4fe922750dd03dc56db552182/coverage-7.15.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:24d8e85a2a45e44883b488c2659f51fa761dad5353fdb319b672a93facbd2ca9", size = 252956 }, - { url = "https://files.pythonhosted.org/packages/a1/f3/52384668c3de4519ca770bf1975a89e4d6eb5aa2faf0da0577a14008cba4/coverage-7.15.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:68931b5fe746ed4fdaa8892989cab9e6c35781eeb3b0ab2ded893d561e1b3652", size = 254797 }, - { url = "https://files.pythonhosted.org/packages/ce/68/54b807e7c1868178e902fd8360b5d4e559394462f97285c50edf1c4608db/coverage-7.15.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:1ce6947e2a95534ecaa5a15e73c21e550514c980d80eda204d064d789a95f6a4", size = 252762 }, - { url = "https://files.pythonhosted.org/packages/7b/48/dde8adf0338e3ace738757dccf1ce817e5fdcadfae77e1b48a77e5a3b265/coverage-7.15.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:841befdbc89b9c82435fc25b0f4f41858b6238693e45af758bec4cfc1968171c", size = 257037 }, - { url = "https://files.pythonhosted.org/packages/07/f2/179dd88cf60a0aeeee16a970ffe250dccea8b80ed4beab4c5d3f6c41ad4b/coverage-7.15.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5d3de58b837375e7f4c0e1a088ccab5f655efb2fd7427b729df02c862a559633", size = 252577 }, - { url = "https://files.pythonhosted.org/packages/2c/37/8a593d69ab521beb6a105a2017cac4ba94425ee0a8349e29c3c0b522d24f/coverage-7.15.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b1801963f9f44ae0c0f6d737bc7aeb2bbcde7d1fe7e3b43cddc1961af42d3b41", size = 254235 }, - { url = "https://files.pythonhosted.org/packages/6d/34/bc9b3bced66f2cdad4bf5e57ae51c54ea226e8aaaebfc9370a9a11877bf3/coverage-7.15.1-cp314-cp314-win32.whl", hash = "sha256:8c7953c4128ef53b6ffb5f90d87c87d4ce26731df294760bb2314eb0e069e44b", size = 223771 }, - { url = "https://files.pythonhosted.org/packages/4c/f3/4d20337bed61915d14349e62b88d5e4144d5a9872b64adbe90e9906db6db/coverage-7.15.1-cp314-cp314-win_amd64.whl", hash = "sha256:6f0bab60a582d415f0fb535ccff13ba334a47a1538f98913330a525d23bd535a", size = 224257 }, - { url = "https://files.pythonhosted.org/packages/7b/df/bbfeae4948f3ded516f92b32f2d57952427fc5ecfc0924487bb6ee6a5f38/coverage-7.15.1-cp314-cp314-win_arm64.whl", hash = "sha256:0f410ee8f0ac4ec7db71bc0b7632a8b9994e1cad2755bd1566c17e6a162caa74", size = 223683 }, - { url = "https://files.pythonhosted.org/packages/35/65/0b431856064e387d1f5cf474625e4a0465e907024d42f35de6af19ced0be/coverage-7.15.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:fc868bab88e049d41fcd41766810d790a8b960053be2a45e060f5ce0d31d258b", size = 222298 }, - { url = "https://files.pythonhosted.org/packages/a6/96/50eac9bd49df8a3df5f3d38746d1bf332299dffb554486c94ebd55c9dc49/coverage-7.15.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:206d4ec6028f2773b40932d09f074539d6bcdd8f6b318d40cb04bdbd68ed0b49", size = 222561 }, - { url = "https://files.pythonhosted.org/packages/e0/5b/6ba1c4a27e10b8816fd2622b98162c83d3bdf1185097360373611bf96364/coverage-7.15.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:620482ef1c9f4e61f962e159325fe77dea59d16e39d9c9470d069053b244d864", size = 263923 }, - { url = "https://files.pythonhosted.org/packages/e4/59/fe03ade97a3ca2d890e98c572cf48a99fda9adba85757c34b823f41efe1e/coverage-7.15.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d385fc9b054e309ad3cecdc77b586d2af0c98aeec2fdb3773544586f366e817c", size = 266043 }, - { url = "https://files.pythonhosted.org/packages/16/e0/55c4b1217a572a43e13b39e1eb78d0da29fb23679003bd0cdf22c50b1978/coverage-7.15.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1198bca9c0dd7c188aae1f185b0c0b5fc4f0a2b6909000858c29550320bdb07", size = 268465 }, - { url = "https://files.pythonhosted.org/packages/b1/e2/ee47944f76afc03909119b036fe9e0da8cbd274a5141287de79791a0fb6d/coverage-7.15.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5d0297e6a070eadb49df7cddd0ab6f420b8b689dd8904c7dd815a323168fa57e", size = 269584 }, - { url = "https://files.pythonhosted.org/packages/cb/8a/6b4d9779c7b2e21c3d12c3425e3261aa7411399319e27aa402dfec4db5d0/coverage-7.15.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:916fcf2214f56960e409561b37fc32a160a42b6e85483d0652d7b70fa55d707e", size = 263019 }, - { url = "https://files.pythonhosted.org/packages/c4/1e/db5c7fa0c8ba5ece390a1e1a3f30db71d440240a80589df28e66a7503c40/coverage-7.15.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f837bae572c7869ffaa502e604c87e182543012831cf87aae4586ad090ac6dcf", size = 265916 }, - { url = "https://files.pythonhosted.org/packages/83/53/fe5176682b00709b13fab36addd26883139d0dea430816fea412e69255e2/coverage-7.15.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3ea65e3ee6c7c32349fd00559927a9e577bdd72386087eeed1c42b62dfce9b82", size = 263520 }, - { url = "https://files.pythonhosted.org/packages/f9/e7/16f15127be93fbc70c667df5ec5dce934fc76c9b0888d84969a5d5341e2c/coverage-7.15.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:345034976f46a1c54bd17f4e43eb30bb92cb7082fcddff03250cff136cc4eb82", size = 267254 }, - { url = "https://files.pythonhosted.org/packages/cd/73/e5119111f6f065376395a525f7ce6e9174d83f3db6d217ea0211a61cca4d/coverage-7.15.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4f051a64eb8f8addb4661c2b41d6eea5b7ebc68ad4b2baea8d9bc54e1956e5f7", size = 262366 }, - { url = "https://files.pythonhosted.org/packages/d7/9c/6d0a81182df18a73b081e7a8630f0e2a52b12dfd7898c6ab839551a454d2/coverage-7.15.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a7625770f7720b49bb30d194ad2f8d50fab3c5177874af3d2399676f95f9c594", size = 264680 }, - { url = "https://files.pythonhosted.org/packages/d9/a2/bac0cbd4450638f1be2041a464b1766c8cc94abf705a2df6f1c8d4be870d/coverage-7.15.1-cp314-cp314t-win32.whl", hash = "sha256:81e503d130a472ad1bd38199ecd35116b40d92bcd31e27a2cacde035381f2070", size = 224077 }, - { url = "https://files.pythonhosted.org/packages/f0/b2/d83c5403155172a43ba47c08641bad3f89822d8405102423a41339d2c857/coverage-7.15.1-cp314-cp314t-win_amd64.whl", hash = "sha256:724e878b213b302ad46e9f2fc872d386613f20ebfc492a211482d917ea76c14f", size = 224908 }, - { url = "https://files.pythonhosted.org/packages/cc/41/442b74cad832cc77712080585455482e7cc4f4a9a13192f65731dcd18231/coverage-7.15.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ce2f05c14d077f406fefc4fa5e4f093ad0e0787549f6582535d6e28766f0361b", size = 224219 }, - { url = "https://files.pythonhosted.org/packages/34/98/07a67cf1a26e795d617ed5c540c042b0ac87b72f810c30c07f076cf334f3/coverage-7.15.1-py3-none-any.whl", hash = "sha256:717d01e6e00bed56ad13306f19e0dd2f4f645ee8159d2c72c72301d6cfc7090c", size = 213284 }, +version = "7.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d0/55fe630f4cf94e3fcba868240fad8c8cdd1f764e2a932f8926347e6ec4cd/coverage-7.15.2.tar.gz", hash = "sha256:3df60dc267f0a2ca23cb7a9ab1109c62b9335ffbf519fcfe167157c28c09b81d", size = 927741 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/3a/54536704f507d4573bf9161c4d0dd3dd59b6d85e48c664e901b6844d8e33/coverage-7.15.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2f1ec6f304b156669cfde653b4e9a953f5de87e247ea02ac599bce0ab2744036", size = 221414 }, + { url = "https://files.pythonhosted.org/packages/b6/d9/8ba925d29743e3577b21e4d8c11a702b76bc93c41e7fdfd1177af63d4b8d/coverage-7.15.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4d3361879d736f469f45723c11ea1a5bbdaf1f6928f0e632c940378b5aa9b660", size = 221913 }, + { url = "https://files.pythonhosted.org/packages/09/54/a855f3aa0187f2b431ade4e4791b77b56282cfb5d201c83ec26a31b5b36a/coverage-7.15.2-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c6a98d698f9e2c8008d0370ec7fc452ebfcc530002ae2d0061170d768b992589", size = 252332 }, + { url = "https://files.pythonhosted.org/packages/8e/d3/13ac97b4370640ba3452fc8559b06cc2f479ce3ba4a0b632a73e44c38a7d/coverage-7.15.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d50dd325e18ec25bfcc10cd7f99b04df1ab9ec76b0918c260e60817ad0643dee", size = 254243 }, + { url = "https://files.pythonhosted.org/packages/88/83/5eca144942d8d0659d3f55176517f4a59cdc65eefd17146a0770935a3ebd/coverage-7.15.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67d7602480a47bdf5b675635403625553ebaa70d5a62a657c035149fd401cea0", size = 256352 }, + { url = "https://files.pythonhosted.org/packages/4e/ba/d3db2e01a50fc88cdb4c0f19542bcf6f61489e34dc9aa3538413e2459a38/coverage-7.15.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cee0f89f4767a6057c8fbf168f8135f18be651300496086bd873e3189fed0487", size = 258313 }, + { url = "https://files.pythonhosted.org/packages/78/b3/aba83416e9177df28e5186d856c19158c59fc0e7e814aaa61a4a2354ad1b/coverage-7.15.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a29ec5305a7335aacee2d799e3422e91e1c8a12474986e2b3b07e315c91be82f", size = 252449 }, + { url = "https://files.pythonhosted.org/packages/6a/a5/4b00ecac0194431ab451b0f6710f8e2517d04cef60f821b14dec4637d575/coverage-7.15.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:48ccc6395958eda89093ecdc35644c86f23a8b23a7f4d44958812b721aad67c1", size = 254043 }, + { url = "https://files.pythonhosted.org/packages/75/b6/cfa209b4313ee7f1b34da47efcd789ea51c024ad35af390e00f5a3c10a2e/coverage-7.15.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:81f382c5a94b434ec1f6da607edb904c76d7212e618cd4d1bc9f97bed4120ef5", size = 252107 }, + { url = "https://files.pythonhosted.org/packages/36/67/e8cac5a6954038c98d7fe7eb9802afe7ab3ecb637bb7cc00e69b4148b56d/coverage-7.15.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bbc808daf4f5cd567af8075ecc72d21c6dfef9a254709a621a84c217c935ebc0", size = 255873 }, + { url = "https://files.pythonhosted.org/packages/2c/92/395cca9f330a86c3fe3471d73e2c102116c4c58fdc619dbbc125c6e93a54/coverage-7.15.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:a4c46b247b5d4b78f613bd89fea926d32b25c6cc61a50bd1e99ba310348f3dad", size = 251826 }, + { url = "https://files.pythonhosted.org/packages/51/60/3e91b20295439652424f426b7086ec5bf4fbe3f604c73eda22b986c4fd6b/coverage-7.15.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:094dd37f3ef7b2da8b068b583d1f4c40f91c65197e16c52a71962d5d537fc5db", size = 252735 }, + { url = "https://files.pythonhosted.org/packages/a5/eb/8c07839005e5e3c6b3877d3a6e2a80ce766589f31dd2b6882b78d59a7b8c/coverage-7.15.2-cp311-cp311-win32.whl", hash = "sha256:a63b9e190711134d581c4d703df5df09851b1acf99792c7aacbbe9f41f0283c9", size = 223500 }, + { url = "https://files.pythonhosted.org/packages/2e/98/59d83c257cd59f0fbaf9d9ddb26b744a576760dfd1ae16e516408894a02b/coverage-7.15.2-cp311-cp311-win_amd64.whl", hash = "sha256:8bb9f4b4279187560796a4cdaca3b0a93dd97e48ee667df005f4ed9a97403688", size = 223973 }, + { url = "https://files.pythonhosted.org/packages/ea/09/2d285c8bef5c4f695d120c1c96dc11715638aa8e134069f210bb6a62a9fe/coverage-7.15.2-cp311-cp311-win_arm64.whl", hash = "sha256:8c726b232659cbd2ae57ade46509eb068c9bd7a06df9fcbff6fe484870006934", size = 223519 }, + { url = "https://files.pythonhosted.org/packages/6a/50/eb5bf42e531611a9f8d272556b1ed4de503f84a91413584094487cf69f8f/coverage-7.15.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1adac78e5abc7c5438f7a209c9ca69d06542f0bf481d728b6989ea80b813fdf9", size = 221587 }, + { url = "https://files.pythonhosted.org/packages/06/d1/da99af464c335d4e023a6efcd7ec30f63b88a43c93745154ab74ffb31cea/coverage-7.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b868acc62aa5de3be7a9d05c2333bf8359ca987e43f9cb30ff8fbda6a024ab73", size = 221943 }, + { url = "https://files.pythonhosted.org/packages/5b/8a/13c42723d61ca447eafa18732e8141dd6a63f2732e1c7e1502c182dd88d7/coverage-7.15.2-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6f6966fc30e6f06ca8f98fb0ce51eda6b111b3ee8d066a8b1ec9e77fa06ab55d", size = 253450 }, + { url = "https://files.pythonhosted.org/packages/d7/29/99021303f98fbdcb63504b4d07bea4cc025b9b2dd907c4f07c85d50a0dab/coverage-7.15.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:68af907f595ab01a78f794932ff3bdf929c316d3000810d38dbc247129e26f8b", size = 256187 }, + { url = "https://files.pythonhosted.org/packages/f9/a8/fd503715ed6ca9c5d742923aa5209257340b367a867b2ced0c7d4ba8a0b9/coverage-7.15.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afa29e2eff3d5729267e2cb2fd4ce9d61c952932fb2694e34ccb5d9540c6a296", size = 257301 }, + { url = "https://files.pythonhosted.org/packages/da/40/3f4b8fb409810036ebc2857d36adc0498c6e957b5df0290c5036b2e143f1/coverage-7.15.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bbf44513ceb1589e31948e20eafbde9deaface90e1a1afa5f5f77b4423d17ce6", size = 259562 }, + { url = "https://files.pythonhosted.org/packages/0b/8a/9bdffbef47db77cce3d6b02a28f7e919b19f0106c4b080c2c2246040f885/coverage-7.15.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9deddf09eecb717b7f980414b43d90a5b22ff3967d2949ab29cb0aa83d9e9098", size = 253841 }, + { url = "https://files.pythonhosted.org/packages/1b/1e/9031efde019d31a06646261fce6dfc5c3c74e951e27a71e5c9a424563178/coverage-7.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae901f7e55ba405c84ee1cab3d3e962e4e871e4a2bcb9c90911adbd69b42ac5a", size = 255221 }, + { url = "https://files.pythonhosted.org/packages/56/db/787acde872389fc84a9ef9d8cd1ccc658e391ab4cb5b28092a714426a394/coverage-7.15.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a0f47002c6eeb7c280228467a4cb0cc15ca2103a8421b986b2d3ec04a0f9bd8b", size = 253366 }, + { url = "https://files.pythonhosted.org/packages/2f/9b/6f57bc4b93c842eef1695f8cdaf2318e35e7ba54f5ba80d84be213ab7858/coverage-7.15.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1cd7a5beb7af3e864a13b1f0fb26efd3695da43ef0daf71e586adfffaf34d5b2", size = 257434 }, + { url = "https://files.pythonhosted.org/packages/88/26/b3186a21b2acc83e451118978905c81c7072c3333707804db09a78c096a2/coverage-7.15.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:97a5c5457a9fb1d6c4e06cfb5dc835871fbfb6a6a51addc9e925bdeff5ef7440", size = 252935 }, + { url = "https://files.pythonhosted.org/packages/20/c2/c9f3376b2e717ea69ed7a6e9a5fcab968fb0b290db6cf4bd9a1fc7541b75/coverage-7.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0901cfe6c13bcd2302da4f83e884555d2a22bda6e4c476f09ef204ba20ca536e", size = 254807 }, + { url = "https://files.pythonhosted.org/packages/f0/e1/dfc15401f4a8aaeb486e1ba3e9e3c40522a6e38bd0ecf0b3f29cb8082957/coverage-7.15.2-cp312-cp312-win32.whl", hash = "sha256:b171bdd71cb7ff792bf32e376173b0ace7e7963e7e57c58dfc42063a6a7174cd", size = 223641 }, + { url = "https://files.pythonhosted.org/packages/91/40/81b6d809d320cd366ec5bdf8176575e897dcb8efe7fb4b489ef9e93e4d13/coverage-7.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:582edc45c2040543fef83341be23c43024a3ab3ae0c2d8bc498a06282905ad40", size = 224172 }, + { url = "https://files.pythonhosted.org/packages/ef/28/9f14ec438149f7de557f45518f09b4a7917b795cc37083aa7db482693f8c/coverage-7.15.2-cp312-cp312-win_arm64.whl", hash = "sha256:a638db90c61cd219aeee65e83a24fdaa57269a741ae0cf773309208ac862cee3", size = 223556 }, + { url = "https://files.pythonhosted.org/packages/fc/d5/f8c838e6b7282976f7c918884b792df7a0c42c5bba5d99c60ad2d221d56d/coverage-7.15.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1121caa19159a38b5463eaae4b1e1fde81e525b15ecc5e000cd5b1a108f743a8", size = 221606 }, + { url = "https://files.pythonhosted.org/packages/bf/37/97c926376364f66298cc44893b89cdf17b8bc406376497c4061ae4b8a8ff/coverage-7.15.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a300c6934e0989c327b9e8a1e110329da4641149f872bbe9f70168be66da76c1", size = 221982 }, + { url = "https://files.pythonhosted.org/packages/b7/30/a36050a6e83c2135ee0776f452ca3948224befc6d7f26acecc082d0c106a/coverage-7.15.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2617f8799d268fabdeef42a7e89ac3a23e1deee9025427db2df970f99a89a578", size = 252972 }, + { url = "https://files.pythonhosted.org/packages/31/d3/06b5f1daf95f0f15ab05bd75f26ba5f3c8b33d0bb72f3aaa3cf41d1bad3a/coverage-7.15.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7dc2950a2992cd676d35c20ae63522836deeb034f08874699d14068710af3dc1", size = 255569 }, + { url = "https://files.pythonhosted.org/packages/81/1c/9afb3f8de2b8d36960391c48559a2e3ff96594b58099f115921549ea8d0d/coverage-7.15.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e36686f7a442185db2400b3df171aac520869faf9deb59df687d28659eda2a6", size = 256806 }, + { url = "https://files.pythonhosted.org/packages/64/d8/b989f96061a5e32d82fddd1b1b9ff48a7c8f8ae7606f0e80fd9de54b1e33/coverage-7.15.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d29ca7bd67af6e12e74632d65f026eabc1364da5c254494cd914446a28a3ef7", size = 258936 }, + { url = "https://files.pythonhosted.org/packages/b8/fa/f99771f5110457c7b511c1935ca49ddf288218eaa84322e028b9334146ae/coverage-7.15.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:db9c8438057e5b0f6a22a0af99c0c1d26b57fbbdbd1be5861ddb8f897fcc3a2d", size = 253178 }, + { url = "https://files.pythonhosted.org/packages/f6/96/c098a6044d119c751ceede7be91035fa8310170ec24a6523aff72f0a5793/coverage-7.15.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:63022c4c8dec1d0342f05c3ede99842fe3d007689acc45e86f123a1746e4a026", size = 254934 }, + { url = "https://files.pythonhosted.org/packages/b2/a2/1457b3a7a50c8d77500103b97a046db863e2f59a1cf6d2f814595f349885/coverage-7.15.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6c0be82b4d4aa5b2704e08518e2252f3e3d110164bcca826816801052e48a7aa", size = 252898 }, + { url = "https://files.pythonhosted.org/packages/6c/0e/76958874c471ecfcdde0d2b2747bb2c61bdbf34a40636f4ce9db9923e643/coverage-7.15.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4510fb9cdf6bb02dfa6af0be4a534b8102d086e22e4a33f8836df663da3d660d", size = 257056 }, + { url = "https://files.pythonhosted.org/packages/7c/7c/3d7c4e3bf58baa40327dc7edc2272b17cf02299366d52763db1b0ca1556a/coverage-7.15.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:42ec3d989421b174a2ab607c1539f24127ad362757b7f1c0c0d7a2993f7eb37b", size = 252718 }, + { url = "https://files.pythonhosted.org/packages/c8/b8/1cecffed9ce14fb25be9ba42d37b6bb61485c9a3ddd43cd3dde36b6087d8/coverage-7.15.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8f91bce78e32343af184c3b7fa28fcf5a9e2641f4b6623d392038f804939188", size = 254490 }, + { url = "https://files.pythonhosted.org/packages/6c/2c/42984561bc7f4c045dca67516a0c50ee5ef8d84352dbeb5559dc86c4823e/coverage-7.15.2-cp313-cp313-win32.whl", hash = "sha256:434e68d531858205895eb0d74b73d20b84260de426387d53c422a5acda2cf050", size = 223647 }, + { url = "https://files.pythonhosted.org/packages/41/9f/39c7c9245efc583beddf89a87683574e663ed93637f3afb6cd7b88405676/coverage-7.15.2-cp313-cp313-win_amd64.whl", hash = "sha256:26c3b04a6377fd7c09800921fa934e3a17c0020439cd59df73e73ae1d4b6a78c", size = 224190 }, + { url = "https://files.pythonhosted.org/packages/c7/de/3a2883cf8a213659280ef4b403059e17a9acaeb7fc7fd4105e1226ff2e6d/coverage-7.15.2-cp313-cp313-win_arm64.whl", hash = "sha256:3ed010aa1b69cda8e827aabfca9866216c980e2dca82ab9a78c5f83689964c8b", size = 223583 }, + { url = "https://files.pythonhosted.org/packages/81/5f/aed265fd7a3551a394f36dfe41868aee709b7f95db4052205b4ad1563ac3/coverage-7.15.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:40f633c5c5fc783732f6312280122e859538fa24461235597c13d803ea9a108a", size = 221650 }, + { url = "https://files.pythonhosted.org/packages/6b/2c/222ba12a545189017120f8eddfc1a0bd4616b47d5d4a8d99421edb2fe4c6/coverage-7.15.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:075560438765b7a2ef43bf7aa7758661b53d889df47f062a31bda6c1ade553a2", size = 221988 }, + { url = "https://files.pythonhosted.org/packages/aa/38/304b5877ab46e6c290b4292cfcf3fe28245f0e5597cad7f6acc91fc7e0a4/coverage-7.15.2-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:25fd15dd40a0a2c51a500d664ca29053c09c3259d998407bf982b6e114696138", size = 253029 }, + { url = "https://files.pythonhosted.org/packages/6c/58/821b533b8db9e44cf1d8a97bd525149ced40dde1d0093da02cb78e715244/coverage-7.15.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b9a6367e4aff723e8ee8190836836124284e8fcd4265e307c844010cfa074f3f", size = 255536 }, + { url = "https://files.pythonhosted.org/packages/f1/f2/7aa06604c389d32ea7f0a6a988359a7eafc3cd3f8e7bc2e88cd2fdf0b877/coverage-7.15.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9854ca62c152874b2060772503535be2e8f53f70b8aaa7686b094888d872f984", size = 256881 }, + { url = "https://files.pythonhosted.org/packages/a2/4f/1ef342339c7916d0096bc5888cc0f653882cc7bc8f897d5cb89143287c9b/coverage-7.15.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:913b6c56e110da40e035bbd168353bf7aaa2544a5eaccea5d98a4629aac156c7", size = 259196 }, + { url = "https://files.pythonhosted.org/packages/fe/f4/7ed055d7a9c5ec13b161773a115a5ccc6b0081d568c31fad830806306cc7/coverage-7.15.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aaccad4129d735a8a4d526f26929894c9a4e8ef7034566f210b176749d6906e3", size = 253036 }, + { url = "https://files.pythonhosted.org/packages/14/79/ea82cca18c242a3a38b6c017da39726aa62dcb64aa635abf79b92009975c/coverage-7.15.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a164b50081fc7357331c4024ef4d17b78ba325f8380d05f5a69599a7e05257ee", size = 254887 }, + { url = "https://files.pythonhosted.org/packages/a4/ba/a136db3c0d9562b00e10b72540dbf3a33cd3bc5b95060c9308e247494623/coverage-7.15.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:bfd341ccf78128e72c094bc70cc25b3ef309c33c7c2c66ba3ed4309549e02de1", size = 252852 }, + { url = "https://files.pythonhosted.org/packages/17/17/ea334246b16b7d059953fad6fdefa11e33c68efbd3fe37b1098120a1fac2/coverage-7.15.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1473b3ba8e7ee0f076117b1a72c23f579a2b9e2bb742f48a8d86ea27ca93f91a", size = 257128 }, + { url = "https://files.pythonhosted.org/packages/ed/c3/074fb66d46d607855f710876b117cbda562c5ab08363528e78820449f937/coverage-7.15.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:17c432b5f73ad52ef46fb06019f6fa7c66ce381961cf0f7dfd1d3a4bd3a98145", size = 252668 }, + { url = "https://files.pythonhosted.org/packages/e1/c1/f620850ada9b36435921c9a3a8057013422b1d964eb4bf37fe138724d192/coverage-7.15.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:77f0ef5011df53a4bd1b35211ab122287f8d9b8d7aa1c4553e5c2deb24b1d446", size = 254325 }, + { url = "https://files.pythonhosted.org/packages/cc/31/a729ca3689404493af82ef8e6ff70bd88bdda8da89aeef6ca9b387aeb2b4/coverage-7.15.2-cp314-cp314-win32.whl", hash = "sha256:f653e5d7248c1191ec988a85c72edeab46c3ff44f90639a4ed4874ec0be90243", size = 223844 }, + { url = "https://files.pythonhosted.org/packages/c6/83/5d809dc808fb1698c671f3e372259bb9158e64b7ea526fc6ab7de64de9fe/coverage-7.15.2-cp314-cp314-win_amd64.whl", hash = "sha256:9911f31aad8906abe337c271343485cf20df5e70df5d2f57f9f136e7b55f26bc", size = 224331 }, + { url = "https://files.pythonhosted.org/packages/16/4e/35e488548e952795829e129995c4174df33bf432b591d1aa42c8d9e4e7ad/coverage-7.15.2-cp314-cp314-win_arm64.whl", hash = "sha256:e38def96ad59853824c97953fdcd2c320a84ba3ce99b417db78af8bb6c3db635", size = 223760 }, + { url = "https://files.pythonhosted.org/packages/ed/49/dd2c86cd6374038f6e415fb5bfb86db5218553209c081384a020369dee79/coverage-7.15.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:835ec4e20b45f0a7f63ed78f94065aca00de033403df8377bfe8b9c6abc0a7be", size = 222384 }, + { url = "https://files.pythonhosted.org/packages/d3/74/173ff17a1c0808e5a438f549f6f145d5ac7528f2791310b63523e3200ac7/coverage-7.15.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7466cc7ab6dc0db871d264bf99e8779f0917ee63d40730af0552f71535a6e072", size = 222647 }, + { url = "https://files.pythonhosted.org/packages/84/f8/b8cba872162356fb44ac79c10309d987206a4461e32072fc29228dad7331/coverage-7.15.2-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e370c12133095ff18432de8c044962be85a5a96d90c6fcbce8e17e76236d2328", size = 264013 }, + { url = "https://files.pythonhosted.org/packages/ee/67/a807a7586d0b8cae485308ddd55756f0806c92f8e0b411bacbf23c48edf3/coverage-7.15.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fe41909c9515c3bfdb5f02c4d1f857dba322d9a9a1178069b91eea77889df63a", size = 266135 }, + { url = "https://files.pythonhosted.org/packages/ce/67/cd78771dc985f7e4ebdcc82b1a96d9a932af9e806f01f2f91a89f4c72e80/coverage-7.15.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6aa28cfb6488e5453b5b762d65f73aa586380f6693a04d58078ce228a29b06c0", size = 268555 }, + { url = "https://files.pythonhosted.org/packages/18/3e/10134cf81275188c58568f324fc74aedff32c63ca4d5bbc513a91944a6f0/coverage-7.15.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bcc0aae933921d03096f53b0b03eeb702129fd406dee59f08d2efacc68681fa5", size = 269674 }, + { url = "https://files.pythonhosted.org/packages/75/4a/771b77de446cba985dc414bbc5844bd21604da05dbc044286df8318a48a7/coverage-7.15.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7c63387e21ab21f512c69c9756a8c7dadd322c7275edb064064433c9a09c3743", size = 263101 }, + { url = "https://files.pythonhosted.org/packages/5f/b5/70a7011da15f4071943361183aefa27847f3e3aec4fd335f1cb3d3a622b1/coverage-7.15.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0e55510bc98ae943cece9e667a6c0fe94c6a92913720dea34243657a17993d0c", size = 266007 }, + { url = "https://files.pythonhosted.org/packages/b4/0d/f9547e804ce7ad49646ffeffac26699510efbe6c0f751b66fdc960c4e825/coverage-7.15.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:2ff08701be2d1556fc78b326c80a3e8042da09352ecb3819105f8e386c8a3071", size = 263611 }, + { url = "https://files.pythonhosted.org/packages/ac/59/f576a396659c0efd351f5c1544f67c3560e89c7761cabf7f65e412beeda5/coverage-7.15.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:38c9518b7103826c403a461544e3c2e77151e8676d06eaed85911a97e962584a", size = 267344 }, + { url = "https://files.pythonhosted.org/packages/7c/5d/c2e4fce3579c0cb635024293f1a32bbe26df101b3e3a69f22243d1352b6c/coverage-7.15.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:dee88b1ed88587abd8c0269a1fc1f4cc77f7750d1dfde2869e2a123af420e67d", size = 262456 }, + { url = "https://files.pythonhosted.org/packages/bb/dd/956287d69436b66094bc4b57ac2da71e43bfd2a5524e958900b9f582fcf8/coverage-7.15.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2fbeeeecea279727f8ac16c8e1133ddfeee793e985c86ae343d6a5ce744eef8c", size = 264771 }, + { url = "https://files.pythonhosted.org/packages/2c/5a/6f979530c2734c575de77cf58f5f28d51f7123a94b5030fd9156fe5f363c/coverage-7.15.2-cp314-cp314t-win32.whl", hash = "sha256:cb0fddaa6884be6aae36ced9544b5e90f7d5f03845a2853bf47a14953a4e8688", size = 224151 }, + { url = "https://files.pythonhosted.org/packages/54/7e/27f6b2a74d484742f4017553e710b01e396b23d809df3e95ca0bb9a2824b/coverage-7.15.2-cp314-cp314t-win_amd64.whl", hash = "sha256:77f091ea3a9cc611cd29f433565476bc1936c084ac8eee00ea0e7e70c27e4199", size = 224981 }, + { url = "https://files.pythonhosted.org/packages/b1/48/284863423aa474240f6842bd00d680da22f4e6ea2e466618ef7c9c9e69a9/coverage-7.15.2-cp314-cp314t-win_arm64.whl", hash = "sha256:6fc448c377d6eeb00a47c673494bd9bae29280ca53987e1869e67ebedfe20658", size = 224294 }, + { url = "https://files.pythonhosted.org/packages/ec/82/32e3bd191d498e64f6f911ad55d14006a0861e54869d2d32452326399e65/coverage-7.15.2-py3-none-any.whl", hash = "sha256:eb6bcae8d1a9d305351ecb108232441d11c5cfe9de840a04388ba5d2db8d735c", size = 213375 }, ] [package.optional-dependencies] @@ -754,7 +751,7 @@ toml = [ [[package]] name = "crawl4ai" -version = "0.9.1" +version = "0.9.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, @@ -792,9 +789,9 @@ dependencies = [ { name = "unclecode-litellm" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ca/95/a307119201f09a5136be1e394282ee4d4d6e3816cbc3c73f370aaa4b5e56/crawl4ai-0.9.1.tar.gz", hash = "sha256:d04251f13ba180107324bb7bbe12029a63d95cc50248f05085743739f38404fc", size = 575249 } +sdist = { url = "https://files.pythonhosted.org/packages/fc/7f/b32541be7b0a4d69574054a07d05c3c36aa1a935914a661b26099db34d4a/crawl4ai-0.9.2.tar.gz", hash = "sha256:58dbfa05a82c1cfa667a20383a1d0f7a42187304da5e4d0661a6f59b0ed6a406", size = 577156 } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/61/fd08ea5870bb7d5b3e028b784fb2e18343cc20b9aabdd4771df35c1a1642/crawl4ai-0.9.1-py3-none-any.whl", hash = "sha256:e154c3219d0cd46b4e06771a725bf35e6c31b19a5c2981afbcdba4574bd0a9dc", size = 479911 }, + { url = "https://files.pythonhosted.org/packages/e0/fe/84960a1747d64d12b4a0d6d9954ee919e44a92dd62e64152552c4e5bf437/crawl4ai-0.9.2-py3-none-any.whl", hash = "sha256:4efb2d0688aa3d66b48721a9031f7257bd2acb52b78d0a89d072741ac685f3f8", size = 480585 }, ] [[package]] @@ -920,11 +917,11 @@ wheels = [ [[package]] name = "defusedxml" -version = "0.8.0rc2" +version = "0.7.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5e/3b/b8849dcc3f96913924137dc4ea041d74aa513a3c5dda83d8366491290c74/defusedxml-0.8.0rc2.tar.gz", hash = "sha256:138c7d540a78775182206c7c97fe65b246a2f40b29471e1a2f1b0da76e7a3942", size = 52575 } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/c7/6b4ad89ca6f7732ff97ce5e9caa6fe739600d26c5d53c20d0bf9abb79ec5/defusedxml-0.8.0rc2-py2.py3-none-any.whl", hash = "sha256:1c812964311154c3bf4aaf3bc1443b31ee13530b7f255eaaa062c0553c76103d", size = 25756 }, + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604 }, ] [[package]] @@ -986,14 +983,14 @@ wheels = [ [[package]] name = "faker" -version = "40.28.1" +version = "40.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "tzdata", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cc/2d/3ead106cef42eab305e15f9c089dcc68a40387d5ce04af18585af4ebbb44/faker-40.28.1.tar.gz", hash = "sha256:2a63fb51abab8790636d4030a094cf942404cbccc9c288d1cad70e2e3e9ecd58", size = 2022717 } +sdist = { url = "https://files.pythonhosted.org/packages/74/e1/adf05724d5838fa9fe2ada64bf390bbf37b77e3714189ae9f17a2c6d0470/faker-40.31.0.tar.gz", hash = "sha256:af163a937aec99dca5abaeb94dd5008c51c26c6e9af1a26c8db4b3c4e7ca4403", size = 2024136 } wheels = [ - { url = "https://files.pythonhosted.org/packages/92/a6/6111b9f13c1564b0e2f5dbeeb611fd00dc731e6add2336f62b798598c73d/faker-40.28.1-py3-none-any.whl", hash = "sha256:e8d3f5c469100a553d246dce7937c291308068a5ec6c9c3a228d7878b50720be", size = 2061052 }, + { url = "https://files.pythonhosted.org/packages/e0/f5/e427b19d79e241a51ce6fcd69a47e7a9ca791a8bc3da00d91d5be05e1456/faker-40.31.0-py3-none-any.whl", hash = "sha256:bedc97c292a48a6a1bbe471a9076d7395a196421ede1cedde99d5719f6b91027", size = 2061930 }, ] [[package]] @@ -1050,11 +1047,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.29.7" +version = "3.30.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/35/94/00f2059e4835eace3ae8fde680b932c496f8ec7bdc99168dfa53fb2e6b79/filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d", size = 71521 } +sdist = { url = "https://files.pythonhosted.org/packages/02/f7/2165ef325da22d854b8f81ca4799395f2eb6afa55cdb52c7710f028b5336/filelock-3.30.2.tar.gz", hash = "sha256:1ea7c857465c897a4a6e64c1aace28ff6b83f5bc66c1c06ea148efa65bc2ec5d", size = 176823 } wheels = [ - { url = "https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51", size = 46036 }, + { url = "https://files.pythonhosted.org/packages/02/df/05118016cad66cd0d7c9417b2d4fc245be35decc4c36810f3c8dbf729d88/filelock-3.30.2-py3-none-any.whl", hash = "sha256:a64b58f75048ec39589983e97f5117163f822261dcb6ba843e098f05aac9663f", size = 94092 }, ] [[package]] @@ -1309,26 +1306,26 @@ wheels = [ [[package]] name = "hf-xet" -version = "1.5.2rc0" +version = "1.5.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/11/21c6756274a6b5a3422f5ec538ccef8f316b2301497097db1418a40e3edf/hf_xet-1.5.2rc0.tar.gz", hash = "sha256:5f96509c18c28fe8182cd56291bc609f86b9213d2277c933f610f3f7049c66d5", size = 899837 } +sdist = { url = "https://files.pythonhosted.org/packages/63/39/67be8d71f900d9a55761b6022821d6679fb56c64f1b6063d5af2c2606727/hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47", size = 903674 } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/2b/54fe48f1ea2708e06b67c09601f63686d90d26b1801af8aa42864ed0b848/hf_xet-1.5.2rc0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:61b67f4dc761b96b0c0acd7a65dfc29c9f4afc8aef06259915aca9769390cc00", size = 4064126 }, - { url = "https://files.pythonhosted.org/packages/f9/2c/d75ae1cc75d0b4dfd757ef62d1ad53ab17038124abc21406be43260bf21f/hf_xet-1.5.2rc0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0d4edc789d5cfa08bf462c59e1c6051bcf8a8b1d8f6846176abb22fcf032b551", size = 3828687 }, - { url = "https://files.pythonhosted.org/packages/e5/a5/2eb5c46b493ceebe3762f498a9369af6fd3366b2032a756de3f26b0925bc/hf_xet-1.5.2rc0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aee03ecf36cf426e644b994350705b9707c1edcf77fa83e7243e74bb77a80d5c", size = 4420271 }, - { url = "https://files.pythonhosted.org/packages/f2/11/3f755f50672097e1c2abbaa22e85ccb11ffcffcc9c5a7b56565fa0b013c0/hf_xet-1.5.2rc0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:1c01c40a855ef218d8ecba0f8895cc5d437daf7f0ab2feaf6aa73ca1b80e5080", size = 4215162 }, - { url = "https://files.pythonhosted.org/packages/d0/6f/2dcda1e46a0015cfca5ee1efd092a20c3e1586c31677f132446a3342e5d6/hf_xet-1.5.2rc0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:82ab79aaf99afb6f69a58393c7f3e4c225b4f8568eb69b3e84bcd599bfa29456", size = 4417013 }, - { url = "https://files.pythonhosted.org/packages/37/4d/a24b0843e492aacbeea0f1027c94eb26d74c8757f491173f37c6038b7739/hf_xet-1.5.2rc0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a62958bdc172b641d69753a24c84d6d5087a285b37bbd3b6da92e85733509c54", size = 4629751 }, - { url = "https://files.pythonhosted.org/packages/a6/95/8c03975cd35a9f5f7f2b38b8f1e5b50e7ed617ec32d803bd5f22b0916c18/hf_xet-1.5.2rc0-cp314-cp314t-win_amd64.whl", hash = "sha256:f751f2daf371bb17334e474b0aa76509b36c8cdb93a65b09ba021f4106a556e9", size = 3998715 }, - { url = "https://files.pythonhosted.org/packages/13/61/df2dbf0ce3c1b717a6aaa59552c36a3f9c73e57677e2c010f40559fb94b4/hf_xet-1.5.2rc0-cp314-cp314t-win_arm64.whl", hash = "sha256:d494b60cf24e65953de4710cd004683e91ff78a34418e58f0f99f0e5ac51f35c", size = 3836335 }, - { url = "https://files.pythonhosted.org/packages/10/c4/38706124609d30b48b3b6717c40d014d3121086e2c53ffa16c9b61c6d3ef/hf_xet-1.5.2rc0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:778abc47410ee7fb26cb4133d292e5e757c144f833e429714221b36d80c2b60e", size = 4057003 }, - { url = "https://files.pythonhosted.org/packages/cc/09/d9f6d4ad6647a8e0f38db343985ba0008cdf76ae900d98e866db94fc39d4/hf_xet-1.5.2rc0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ca1534e791e222ef3c05bfd7b184d81790092b4bcf4ea2895f7c8589f64f4fa", size = 3841422 }, - { url = "https://files.pythonhosted.org/packages/f4/fd/5d839b7f163d9dfff79b05935e8b38957dd02a33d47d73aa29f205c99a8d/hf_xet-1.5.2rc0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:52b62a30f76d94f88e67eefe9ebdc92efb66add83e17dc9bbad6672d2e8776c7", size = 4431104 }, - { url = "https://files.pythonhosted.org/packages/45/72/5f31f5d467bf92b8bfbcc4d9ce9506a9201e58e5e02b407a4c6d0b33834d/hf_xet-1.5.2rc0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b4cd10a7372b82e19b8efc7aa4669e3f3a72111f8900313452baedb59591fe4d", size = 4224712 }, - { url = "https://files.pythonhosted.org/packages/19/86/70c4d6972f494060808eb37d458aaa8d87eedcbc3f0ed8f5e3060449de7e/hf_xet-1.5.2rc0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ffa13d8a2e169cc333745db5c17de83d38d5ac8ed24c0fd98f783b82bdfcbf38", size = 4427711 }, - { url = "https://files.pythonhosted.org/packages/f7/f1/e4fb515d9aae8b1e6160b16a647df7b87dd69223145846aab34a6aa5e915/hf_xet-1.5.2rc0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e393985c5eb0e9bcd3bc8431f7e1c52cd075d4492b2408bee6b89a435ad207db", size = 4637470 }, - { url = "https://files.pythonhosted.org/packages/40/53/fb1a79510b028a9f67bf23726f4f886a8195518dce3768a16150cac8ead1/hf_xet-1.5.2rc0-cp38-abi3-win_amd64.whl", hash = "sha256:9f220ac6b52512764455ecf54dcb74265958c1890f66c00e81fb291991543aa8", size = 4007544 }, - { url = "https://files.pythonhosted.org/packages/2c/ed/7fe6051ca4ef5e81977d44b6d9fa80684b9765c99fb351177ef109a4847c/hf_xet-1.5.2rc0-cp38-abi3-win_arm64.whl", hash = "sha256:37fc59721ca889e4eb1a4b746efe46747752d4a2a0aa4a8763ce43d7a413e569", size = 3842732 }, + { url = "https://files.pythonhosted.org/packages/29/be/525eabac5d1736b679c39e342ecd4292534012546a2d18f0043c8e3b6021/hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b", size = 4064284 }, + { url = "https://files.pythonhosted.org/packages/c5/3f/699749dd78442480eda4e4fca494284b0e3542e4063cc37654d5fdc929e6/hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576", size = 3828537 }, + { url = "https://files.pythonhosted.org/packages/22/d7/2658ac0a5b9f4664ca27ce31bd015044fe9dea50ed455fb5197aba819c11/hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4", size = 4417133 }, + { url = "https://files.pythonhosted.org/packages/d9/58/8343f3cb63c8fa058d576136df3871550f7d5214a8f048a7ea2eab6ac906/hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4", size = 4212613 }, + { url = "https://files.pythonhosted.org/packages/0c/33/a968f4e4535037b36941ec00714625fb60e026302407e7e26ca9f3e65f4e/hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380", size = 4412710 }, + { url = "https://files.pythonhosted.org/packages/7b/d9/9e33981173dbaf194ba0015202b02d467b624d44d4eba89e1bf06c0d2995/hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577", size = 4628455 }, + { url = "https://files.pythonhosted.org/packages/e9/4b/cc682832de4264a03880a2d1b5ec3e1fab3bf307f508817250baafdb9996/hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e", size = 3979044 }, + { url = "https://files.pythonhosted.org/packages/ea/09/b2cdf2a0fb39a08af3222b96092a36bd3b40c54123eef07de4422e870971/hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e", size = 3808037 }, + { url = "https://files.pythonhosted.org/packages/de/ba/2b70603c7552db82baeb2623e2336898304a17328845151be4fe1f48d420/hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799", size = 4033760 }, + { url = "https://files.pythonhosted.org/packages/60/ac/b097a86a1e4a6098f3a79382643ab09d5733d87ccc864877ad1e12b49b70/hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed", size = 3841438 }, + { url = "https://files.pythonhosted.org/packages/d3/35/db860aa3a0780660324a506ad4b3d322ddc6ecbba4b9340aed0942cbf21c/hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c", size = 4428006 }, + { url = "https://files.pythonhosted.org/packages/af/6b/832dd980af4b0c3ae0660e309285f2ffcdff2faa38129390dbb47aa4a3f9/hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025", size = 4221099 }, + { url = "https://files.pythonhosted.org/packages/9e/05/ae50f0d34e3254e6c3e208beb2519f6b8673016fc4b3643badaf6450d186/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f", size = 4420766 }, + { url = "https://files.pythonhosted.org/packages/07/a9/c050bc2743a2bcd68928bfee157b08681667a164a24ec95fbfcfcd717e08/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097", size = 4636716 }, + { url = "https://files.pythonhosted.org/packages/e9/f8/68b01c5c2edb56ac9a67b3d076ffddcb90867abaee923923eb34e7a14e76/hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65", size = 3988373 }, + { url = "https://files.pythonhosted.org/packages/39/c6/988383e9dc17294d536fcbcd6fd16eed882e411ad16c954984a53e47b09c/hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d", size = 3816957 }, ] [[package]] @@ -1826,7 +1823,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12" }, { name = "typing-extensions", specifier = ">=4.5.0" }, { name = "uvicorn", specifier = ">=0.30" }, - { name = "xberg", specifier = ">=1.0.0rc25" }, + { name = "xberg", specifier = ">=1.0.0rc29" }, ] provides-extras = ["litellm", "graph", "crawler", "cuda12", "release"] @@ -2656,7 +2653,7 @@ wheels = [ [[package]] name = "openai" -version = "2.45.0" +version = "2.46.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2668,9 +2665,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/78/60/d4219875289b11d2c2f7da93c36283da224a2e55865ed865ab64e0ce9217/openai-2.45.0.tar.gz", hash = "sha256:10d34ca9c5643bce775852fddbfc172505cb1d4de1ccd101696c3ecff358765d", size = 1109653 } +sdist = { url = "https://files.pythonhosted.org/packages/af/ac/f725c4efbda8657d02be684607e5a2e5ce362e4790fdbcbdfb7c15018647/openai-2.46.0.tar.gz", hash = "sha256:0421e0735ac41451cad894af4cddf0435bfbf8cbc538ac0e15b3c062f2ddc06a", size = 1114628 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/b0/2291689e3ec4723fbf5bbf3b54afcd7b160f9ddc98ca7aedfd0132af5677/openai-2.45.0-py3-none-any.whl", hash = "sha256:5df105f5f8c9b711fcb9d06d2d3888cebc82506db216484c14a4e53cdf651777", size = 1629470 }, + { url = "https://files.pythonhosted.org/packages/ea/7b/206238ebcb50b235942b1c66dba4974776f2057402a8d91c399be587d66a/openai-2.46.0-py3-none-any.whl", hash = "sha256:672381db55efb3a1e2610f29304c130cccdd0b319bace4d492b2443cb64c1e7c", size = 1637556 }, ] [[package]] @@ -3136,7 +3133,7 @@ wheels = [ [[package]] name = "pydantic" -version = "2.14.0a1" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -3144,112 +3141,111 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cb/71/0ae6d4e0a84faf0cd050033416509c44a19dc6dbec5bfdd3e1318cb1feb4/pydantic-2.14.0a1.tar.gz", hash = "sha256:2c3a5627d48f59725564c41b582328a29fa8d9b1108128ef6710463a0136d4fd", size = 844232 } +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775 } wheels = [ - { url = "https://files.pythonhosted.org/packages/91/4e/15d81754895da3f256273432da60fcb7c885177fefaf495c5b4364fadca5/pydantic-2.14.0a1-py3-none-any.whl", hash = "sha256:61a1ea8d65df95b681c1fab9cd7d01b2472837f798df53dc6d0f41f0c217b061", size = 470439 }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262 }, ] [[package]] name = "pydantic-core" -version = "2.47.0" +version = "2.46.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a7/74/319859f70c733f341df03823c8ca27ce9003faaac3ffa3110f3af1c8641a/pydantic_core-2.47.0.tar.gz", hash = "sha256:422c1797a7864b2a9a996435aba92fe571fb80190f67a31edbc1ac040c7b51fe", size = 476601 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/e0/0da7149bbd84c438beb208e17174d03a798f2c8ce5a978cbc19fb3052be6/pydantic_core-2.47.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:fe87ccbc39a103709d0a5afa75240c15a94611af129261e9484bef0bc97960b2", size = 2114007 }, - { url = "https://files.pythonhosted.org/packages/51/30/362c68ac4dc1a830066bf250850725d95be0567bb6a85434616ac68f5939/pydantic_core-2.47.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:45882c24324f123037982c65eb8d60da778447e6bc87c82241f81d6c6d2c307e", size = 1948028 }, - { url = "https://files.pythonhosted.org/packages/60/d6/1dd26612212fab9145a2b0d876459764ad04248331cb6f61ab4909fddc43/pydantic_core-2.47.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:983e39b39547772543f3518557d0a86dbb3b7bb58bf8e82faba1e0cfa3e816d9", size = 1972491 }, - { url = "https://files.pythonhosted.org/packages/ed/c4/00b31fd04acf601efff148add88cfe5f3c21f43f872e79f76e4580cfaf18/pydantic_core-2.47.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3eb92447d3a079b945b61b8cbd6c3ec2954de3655c4efa0ebd35b069e472c2a9", size = 2045790 }, - { url = "https://files.pythonhosted.org/packages/8b/33/7d1a8116bde9d259b3289090a2d93b87562001731cdf81d78bef5d1060d8/pydantic_core-2.47.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af9600c680bec4b8d23c32ddaf7a5d91ed39a2cf758c082e34e860140cdcd87", size = 2223039 }, - { url = "https://files.pythonhosted.org/packages/cd/f6/4c94adcfe1bdaad3f8865a2115bed02e02cba809ffab25c0f85c0cc9b78a/pydantic_core-2.47.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9d6098342d4510a9034a500a53b1d737daf9cfc18a47cd21047d02d7d1587557", size = 2282031 }, - { url = "https://files.pythonhosted.org/packages/b6/4f/7fe179b37f887297a5bfd7b38a4a26cd485748b6b424c43c73a36c02f57b/pydantic_core-2.47.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9a891c20be5110deb1904f639f3615ec5022b3495995850d1abe7b8fa1550b5", size = 2088600 }, - { url = "https://files.pythonhosted.org/packages/fa/15/daef331c1571ee986e1410511218c4d58e6ead75e0db735051925caa572b/pydantic_core-2.47.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:13990d357a50078e382b15fa3ce3f08043223b4be3eaeb340b184f54c1a2397a", size = 2112087 }, - { url = "https://files.pythonhosted.org/packages/f3/c7/7cfbb736f9ac501a92171e93bf640d065a13e12dcbcfb7effd21cce2f1ca/pydantic_core-2.47.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a0f7343305387bb5884f24d384b7978ad099a277b27529e592c041a502a37c32", size = 2171199 }, - { url = "https://files.pythonhosted.org/packages/5f/03/48e63b1f7d2088d747dde98b5dd7b89a46fff9173b6c10dc6189289c4aa5/pydantic_core-2.47.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2c5e11bb1be2de2707c9367f364e73091ef30d34be54b3a4564d7421ac1a16bd", size = 2176850 }, - { url = "https://files.pythonhosted.org/packages/15/8b/e653f7723f1dff5c054dada5f5d7f2c294067c087a52188847a0414cff2e/pydantic_core-2.47.0-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:e1b52aa981e034896712460c899ee30707c8c6a385e79bb7648aac76c748a3da", size = 2315368 }, - { url = "https://files.pythonhosted.org/packages/fd/ff/5aa6fcde3bd427648fee1a363671d8b4852c716603acd77091fb4cf1fc3d/pydantic_core-2.47.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d57d46021c20d4efc28f69b4ca4670dffbb7bdafa51d1967b747849726ec643b", size = 2351750 }, - { url = "https://files.pythonhosted.org/packages/85/1d/6ee14bea4426d6bc421033d8d6ec69a6938d5773f7bd1d535b221fa48a7b/pydantic_core-2.47.0-cp311-cp311-win32.whl", hash = "sha256:d93c02ae8bd33f73624319d85cba47e754155c5bf104c0c5ca96fcd1f3094939", size = 1971464 }, - { url = "https://files.pythonhosted.org/packages/cf/6b/e466253efba866851af6e1094c488b4c5fb6adbfd8993d13e12b2a575541/pydantic_core-2.47.0-cp311-cp311-win_amd64.whl", hash = "sha256:859ca679f00e5feb11b58b616eb7bc0efbb13654be21f5c898e510e27671c900", size = 2070625 }, - { url = "https://files.pythonhosted.org/packages/c1/27/10628ab2df9f2421b6f3b88add9e026f1cb79198be417ee7d684ef0de5d7/pydantic_core-2.47.0-cp311-cp311-win_arm64.whl", hash = "sha256:482667e5b7a3e97b0836f33a716199c4ec6ba9c896ca4db6eae799ec527c1e64", size = 2052484 }, - { url = "https://files.pythonhosted.org/packages/19/23/2daeff7346433ad9a3567852dd7a716329f9a7952dcf1ee74b166271bdc7/pydantic_core-2.47.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:70a7aeba54854f5d97da65cb1a61f000f53df3704cab41cd81d65ab127ddd031", size = 2108216 }, - { url = "https://files.pythonhosted.org/packages/6c/70/2989cb5112b892b7dc13af570ff57d0f383f770fc88bbb644262df1b3017/pydantic_core-2.47.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c0e97329e38228f57fe1f2d91ba0ef39cc75cc1a84fe6ef58942d2fc6cb406bf", size = 1951502 }, - { url = "https://files.pythonhosted.org/packages/fe/a6/2417d8b1856cf7579a21e2679f02417b910e4f29120303e2c45137525072/pydantic_core-2.47.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:faf83e50714837af72f13e9369c50377552a4a74049d4477bed51c7e5822d94b", size = 1975590 }, - { url = "https://files.pythonhosted.org/packages/fc/71/0cd0813355b385bef8fa9a719982e0b44847afedc043b988c9f7c5d02ca9/pydantic_core-2.47.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f50abe60347bf8afe2b2f58db86cf3ac6e418eec7ffa01d9dc90ba29fc64f243", size = 2055885 }, - { url = "https://files.pythonhosted.org/packages/a3/33/5687c6dcf8ecca106c95b2a49f04a633320ab49961da6e57682db37b4482/pydantic_core-2.47.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f13b18e7dec056336f29ee77dea3cc5db0271d6215cac7249cc5c61b0a49d293", size = 2235292 }, - { url = "https://files.pythonhosted.org/packages/36/98/08e81f1a0f0aa82cfa1a4d07b66eb116740fd6c82ba4baa8f96d4b377132/pydantic_core-2.47.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3aaabbdbbbb8dff33fa053ffb2c980f39dec745fc03592f50e1e010449129841", size = 2308945 }, - { url = "https://files.pythonhosted.org/packages/04/f1/6e35bdbdb79f96e630265297452d709b2ebd5facfbd83d3eb702eee24939/pydantic_core-2.47.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d2907ee6d15cf26787bfbeb4c42e18e52f358086eab91baea961a0d909248d6", size = 2093861 }, - { url = "https://files.pythonhosted.org/packages/1f/c4/6a48e5da6a567bcf6d3ab113d2d476e1b93e3be941b4c486918f1c9d6714/pydantic_core-2.47.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:c413761260967f4dbb51135e1b49f30a1c29e15bf371fbb39754ed6475739545", size = 2124948 }, - { url = "https://files.pythonhosted.org/packages/94/a1/e356c98dfc6bd5e6e166f2c832865a62941370b17a7199d35bc5999fecc8/pydantic_core-2.47.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2bb527ac6e5a9721023b24615e1a55c01f47c60007f08b7d2afb89ff9c7a0e22", size = 2182065 }, - { url = "https://files.pythonhosted.org/packages/63/aa/6e99aac0a891b05cc522cc464aebeec2bb877fa4744be641a013ebc1785a/pydantic_core-2.47.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b3abf7fd5e6abe483a63413f9cd26b7c93c20780e19c8556434c7279f6b2f10c", size = 2185830 }, - { url = "https://files.pythonhosted.org/packages/99/68/c6c5c1ad40b4d58f11d6e754e9c3990a2a3cfff20a9cf5c1f0a855a791b2/pydantic_core-2.47.0-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:9cefe43d17a5a273d71697c084d3787defa7f578cd5fab4cef4c66d13c9e44b2", size = 2327330 }, - { url = "https://files.pythonhosted.org/packages/e0/f8/064747538297e385a1f197f35551d1bf6093e88c5ce9b4e15c035f0a1cfe/pydantic_core-2.47.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:38a8b5371b7938e4f6c31060dbd51d3b3229aef7c43eaac2eb8b153e43c3f189", size = 2366505 }, - { url = "https://files.pythonhosted.org/packages/d8/d5/2d8715ef982c0b1875e42f4248b48439133a2781f0be77f0e4dbc84d147d/pydantic_core-2.47.0-cp312-cp312-win32.whl", hash = "sha256:b87e95e644df2a36bd631dd0d6e097aa73d19a55adf7b1724ebdeece3d9c76b7", size = 1957410 }, - { url = "https://files.pythonhosted.org/packages/0e/ea/9d908a80798db41c7c38926259ed74d015869f0afe4be6ed56f71793a084/pydantic_core-2.47.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0078c5695322050ccedbc86eafaf3e2548439782c51d99e575de0e31b9fe4f4", size = 2072354 }, - { url = "https://files.pythonhosted.org/packages/fe/78/5549cba47c662ec7f05d79527474786e5ff7f08ef3e65a409a8a2ab6013e/pydantic_core-2.47.0-cp312-cp312-win_arm64.whl", hash = "sha256:7eedc31996e9eba3bdfbbc380805ac6d765c889b7e93b17cd00ecb0200fd6dca", size = 2035452 }, - { url = "https://files.pythonhosted.org/packages/7a/91/810cabda42f7fdd13b349702694e1140f6071fd42d1e542d606d5ad0c2d4/pydantic_core-2.47.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:263560ece98bffbbc0a8047ce60b8a278c859db6a2a4e30d9454b02891045eca", size = 2108548 }, - { url = "https://files.pythonhosted.org/packages/fa/87/f444a6d63bcbdff3c45291df842941ed56c568f45dd3742f25afb567b5a2/pydantic_core-2.47.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a7fdaff39a66bf66e9037da482575513d2f20bfb02ea9d9222b5cb3b902fc695", size = 1951421 }, - { url = "https://files.pythonhosted.org/packages/f5/09/c90dff5407c11e59023161c84df19dd5ed93966a23b2f08cd21d45812ab9/pydantic_core-2.47.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3e6c8ee5ce8c270bfae09763ae4bbbccfe81090c97d670a621fb86cb1ef6042", size = 1976554 }, - { url = "https://files.pythonhosted.org/packages/95/cf/55fbe9f1b396f91005cd1cb7acf2bcf380a1f1b57e261ac78bcdc0ff42f3/pydantic_core-2.47.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e38cdae682cfec4b3816722dccf6376ca59049726d57dca83c2fe7cc13665589", size = 2055005 }, - { url = "https://files.pythonhosted.org/packages/1b/c6/4f7cbaa62582e95c080cdd45fb5d4350ff05dac87ea32930c75dcb427bf1/pydantic_core-2.47.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc3193ff0b7e2e168f84c6185e70475738c191f3154e0af8f897cd0f8f9a489", size = 2235489 }, - { url = "https://files.pythonhosted.org/packages/31/75/16913c82ffd194c537222b3d0e8a05c11681e551add67308ec4af6023724/pydantic_core-2.47.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57ff41672a615f38af528ee904602be51c653248354e5db8e9252668abe91e68", size = 2308360 }, - { url = "https://files.pythonhosted.org/packages/05/9f/b24bb1b764fc360adace5df806a81fd62ef1662df2973891e487c3fd5a2c/pydantic_core-2.47.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:473b9a2b2a1f0dd55cbb32d2b902f93babe7f141a0bb48fb4d3d4d2b3e93e9a0", size = 2092549 }, - { url = "https://files.pythonhosted.org/packages/4a/d3/0268ea5972b91178250c0a573bb54c17f697665cd2aa35623087a3589fc6/pydantic_core-2.47.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:195f9c4ac43a7b2a044a7b86631c3352abdb820bed2823ea29f98f779255f459", size = 2123849 }, - { url = "https://files.pythonhosted.org/packages/8f/92/26b2147738a89f78925775bb4b34ae53f981ce880ee77ee2c2ccbc49466a/pydantic_core-2.47.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6c1cd10d39ef1ff8bcd68b6865bee9c434631ac0608d402fe86e678851c2e2a5", size = 2181733 }, - { url = "https://files.pythonhosted.org/packages/d6/fa/8a858dde4784c7245b7216ab9a71518cb1a898a83ffb5c0509181789de19/pydantic_core-2.47.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7cbe66352fe2b39511d49150e5b52159429cd21f5633a3e801dd2c43829dcdca", size = 2184065 }, - { url = "https://files.pythonhosted.org/packages/69/20/ea165877f965622e021f04d63330f9ceb55ede0aefa862863e06a9a610cb/pydantic_core-2.47.0-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:e35192a1d53e55d510d8bb1023c988c7cdae6d94539074971741b2a7656e49a1", size = 2326881 }, - { url = "https://files.pythonhosted.org/packages/7b/d6/ad22a6f0941be5a5478e6c378d0ec3c7641be96d7a86a3ea1f9e9830a269/pydantic_core-2.47.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:81de54576de2e20baec76cc5afae2820f9049e6fdc4f357bac3391da02d0ba97", size = 2365574 }, - { url = "https://files.pythonhosted.org/packages/e2/4e/eb5e3429dd29311e75ab89313cde09b709b0637fd88838d2cb4a98760b02/pydantic_core-2.47.0-cp313-cp313-win32.whl", hash = "sha256:05da6647bdfd3888936ac10aa39b239d659f3c93dff281af0fc5943eb55629dd", size = 1955451 }, - { url = "https://files.pythonhosted.org/packages/15/12/ec107c12aa8729c766285d4aa6caa5f5addf8b2ef6cfd35c455a3ca5c66e/pydantic_core-2.47.0-cp313-cp313-win_amd64.whl", hash = "sha256:021220e0a03b66112737ee1fc49759340ce8fafb8d9ade1b7fb366b06033fa45", size = 2071285 }, - { url = "https://files.pythonhosted.org/packages/f4/02/b003a55acfeb35823925d1cd502b80e3fe6c5d477865c36815fbefdf49ac/pydantic_core-2.47.0-cp313-cp313-win_arm64.whl", hash = "sha256:55156ee2f6f561ea4e25ab55f84bd70b9c9ed2546a834cb2b038fe10225aaa37", size = 2034804 }, - { url = "https://files.pythonhosted.org/packages/ef/46/a4675c783822e229bf04e290ae28cf5a057b00a46039498743b066e0fefd/pydantic_core-2.47.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:6e37a6974fbd8fa7cae12285a76970d50b3689ffd6ed7c7fdd176ba81dd22d0e", size = 2104537 }, - { url = "https://files.pythonhosted.org/packages/67/bb/f6d6d1dd8362d616b227b55af8bc2d1b7d4ea842facb7c50a82298ba3b9d/pydantic_core-2.47.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2433b8524785cc117e602233bc574879bc8d87f09523edeec51665d5c46cf42d", size = 1951581 }, - { url = "https://files.pythonhosted.org/packages/3f/ae/7d01ed8658ccf8beae8a3fcf2cd023c3ba0ab0b19d3f456845a709cece94/pydantic_core-2.47.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8f9f2be064c8bf1189f46f7062fd42765d94f59cfb7db7ef8db19563192110a", size = 1978461 }, - { url = "https://files.pythonhosted.org/packages/80/e5/dd38ccbd1e68f1c3fd7f8b413a8b32db11bd16d5999c3dc3aed4e54290df/pydantic_core-2.47.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1f0a9659a2eb161573418e3138f616101ba21bbd2ff04916dca7b6712155e015", size = 2049444 }, - { url = "https://files.pythonhosted.org/packages/7d/63/0cb998f3f4ea4255ab6d891e2537d4566cd31630df0d406ce45b00306729/pydantic_core-2.47.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2995074b99242aa28991e0120a3c881babc139e08750a05b7ea7d140644e091d", size = 2231772 }, - { url = "https://files.pythonhosted.org/packages/3f/20/2cacf5c4a1bdda1356351df38f343c6cf13af6194e6e0ee0f7967395793f/pydantic_core-2.47.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c5b224dc04c3ff9b08c24419464eb7f6ad7a1049e12284a00bf80df82bd15fdb", size = 2305322 }, - { url = "https://files.pythonhosted.org/packages/e3/8f/b7e09086bd48b1ef3f74227ed98907496924a622c7f9315cf661946233c6/pydantic_core-2.47.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:264361b7236d4374fef6342908f87d084a0d58a2f8d0811e99f714309cb0ba7e", size = 2097748 }, - { url = "https://files.pythonhosted.org/packages/bd/27/2927049e93f6fbbaa838b2bb656dbd63c6d6fcaab9909d632fd93573ebd9/pydantic_core-2.47.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:53368beaf693f6302a6e33bdefe950857534a04d282811421bd20176d0fb5636", size = 2122971 }, - { url = "https://files.pythonhosted.org/packages/f2/3b/80b58a0c7f4339f024ad5c5b6316bef895536abf75c45dc85f0c83ae1a92/pydantic_core-2.47.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5d2177b44ba7d9d86850f865f362feeaac6a2ed8517a9b505b97ff0b7fdbd7dd", size = 2181287 }, - { url = "https://files.pythonhosted.org/packages/5d/0a/dfcb9575603aaf113d2cae1fb622570046a210276eba2267764b3f83f4f2/pydantic_core-2.47.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:fb57a6b538ec7a01b937986bc093aec530fb056135b6bc9cfdd0bf8460c25bc2", size = 2177842 }, - { url = "https://files.pythonhosted.org/packages/57/c2/31a198c8180b417d18410e7640c505b39c51ac291aca34f71ac37093f4c4/pydantic_core-2.47.0-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:18c9c7c3a18e9bdbf1215d913f6bd00e17595dc92949817935cb87a3cf5f1697", size = 2321802 }, - { url = "https://files.pythonhosted.org/packages/11/2d/052d872b61f88d2f71b3d881c08abfbe33cb9d64bd988712bb4fb973001c/pydantic_core-2.47.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2f72a382886ca85bb1247303b9134cf9978c9d454de62e710a1ebcd9d2131927", size = 2363870 }, - { url = "https://files.pythonhosted.org/packages/be/60/c8c985d4081dafaa88fe6ac6a41ebe2b82d289c391b2bcbd3508f585e7a9/pydantic_core-2.47.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:cdf4dc2cdd0eacad1bd81c4d25422b4c25b206acae095d2d64e5d5cb7facc6b3", size = 1369335 }, - { url = "https://files.pythonhosted.org/packages/42/70/30dfcc18dc5552f7235ae2ffd7b699a1ba0d38ebfcdb97f371c792d61601/pydantic_core-2.47.0-cp314-cp314-win32.whl", hash = "sha256:1e859dd5e06e9807080e14995db131649a77c61131cc464a7fe492a69ce82488", size = 1952107 }, - { url = "https://files.pythonhosted.org/packages/fa/49/f5c517ba99b9e1036ead16cedd0fc189dbcbf900fb0b85b746b6a0b9b389/pydantic_core-2.47.0-cp314-cp314-win_amd64.whl", hash = "sha256:234ecade0e358caa1ea516c218b3f61e61e30532cad1a8bb12f2487325838548", size = 2071388 }, - { url = "https://files.pythonhosted.org/packages/eb/6c/6b95e00ed7a3c7f78cb8544bc34cb29412c05512af507fc4466912576d50/pydantic_core-2.47.0-cp314-cp314-win_arm64.whl", hash = "sha256:58158d0111e86893bc35aacabe509f951ed303cddf8cdba43533190bde317914", size = 2026046 }, - { url = "https://files.pythonhosted.org/packages/22/2c/fa67f96f381ba3c83e8cd75a6eb3c538e123d8d02e19d80383bff01ffda0/pydantic_core-2.47.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:24017fd3befd4d7cc4a8c71f4a1e9a44d29fdc91723c5446b0e795ab808adee7", size = 2102407 }, - { url = "https://files.pythonhosted.org/packages/9b/e8/99abe6c33db3dda235a2095eae5fdd03266857abd9be1c50ed97a59587c3/pydantic_core-2.47.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:dfa0820888cc4549fcce7e6bd8affa7d75198d885ccd0bb0760def4bb8461862", size = 1931963 }, - { url = "https://files.pythonhosted.org/packages/5c/dd/96d1884f65737f793e64b6b8745ee793e3cb4735e086ead0a40e5349294e/pydantic_core-2.47.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1d507f331756f7066cde7c9f35ed78fa78223a54369dbc8a34d6da7a5074fa1", size = 1973492 }, - { url = "https://files.pythonhosted.org/packages/07/8c/0504f091c75b229cd07d61b45464d7c5291c3ecf2e7d847f215fe81cf5e5/pydantic_core-2.47.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5c4a885fb50c05903bc703a00830616e680304fdcdd90fc9535a52e72debe712", size = 2035548 }, - { url = "https://files.pythonhosted.org/packages/0f/bf/e9f69c55ef6c945d29e4afb306f62a15354bfd09f1ee25d9aecd4c6cbb82/pydantic_core-2.47.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7b107cbd764bf68f12a57c7aa5846d868bc7463490a1ac2d0f19bffef624c5a9", size = 2238356 }, - { url = "https://files.pythonhosted.org/packages/b3/1d/dfbd214487033093f5946c1fd5915ae5dc6b278efe764397b5e2ef8092a8/pydantic_core-2.47.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0b5f46412c8226d0c8f1f423c324c75afe342e4b854836933579fb484f68598c", size = 2284799 }, - { url = "https://files.pythonhosted.org/packages/f1/ec/f80488fa26a6b27bd0d1267f35b0fce3bfeec23ca97021085a94ccd3adc3/pydantic_core-2.47.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90cbf7e35d597503cbdb5cd85409cbb75f377290bc7e8e37cc5dfe4f5cc66cf8", size = 2107895 }, - { url = "https://files.pythonhosted.org/packages/9b/1d/67ac210745f63a982b145d8f281dc76bd347d98052792f087224a178da54/pydantic_core-2.47.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:04ee7ba7172cb4484af51b2890f19069d35773698abc8c6ebb651f52fbf41134", size = 2102797 }, - { url = "https://files.pythonhosted.org/packages/6c/75/f3b9bd4d88fcaad9a5abf2ec6969f37a70e34f40a5f8d2d78077f168e83d/pydantic_core-2.47.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2a7acb3e120ddba94372b8146e62ea3a0bce203180e34641c817c87f995c91e0", size = 2160123 }, - { url = "https://files.pythonhosted.org/packages/13/1b/e94ac13bd17c341e86c4b67deb47b63718208fe413034fdb01e7a4bf1dd8/pydantic_core-2.47.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:1a2f7ceb58013d167d8c96f10ec9b3a137018c819ba356f68ff1cb74302fd22e", size = 2164386 }, - { url = "https://files.pythonhosted.org/packages/36/e5/862ff195402f5b08db9ad4d9ebe7cbed993f51528aab7edcb62531442556/pydantic_core-2.47.0-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:482b097637fec5037eb13fe9f9d5fe47e568a5b451d686bc2b854076cc0b50ca", size = 2303767 }, - { url = "https://files.pythonhosted.org/packages/c2/05/c8ebbb97cf44686fc9cc64a6bb86ac72a8871426a9f61417e395d77c4894/pydantic_core-2.47.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a8c7f5fad73eb404f4b84c75f3d9d3865b748ded248b7366341db6e516fc502b", size = 2361148 }, - { url = "https://files.pythonhosted.org/packages/c5/8b/4695ec6ce81eda7cc822383de993c5b2c289cfc84d3f032a30c55eb238fc/pydantic_core-2.47.0-cp314-cp314t-win32.whl", hash = "sha256:f343c39928097175acf2f7d0cba5c00b0f62265d88a173a4ce264266ac849bd9", size = 1939588 }, - { url = "https://files.pythonhosted.org/packages/96/4d/bc2e188ab648b3ab284e80a0340c5d5b9723c22572c6e36ba39eb300e718/pydantic_core-2.47.0-cp314-cp314t-win_amd64.whl", hash = "sha256:52d40e074da44e42b2425aedebd513e405a31807036ef597175117a9b01743a6", size = 2049697 }, - { url = "https://files.pythonhosted.org/packages/6f/46/ca1e6813e029993e57340c6a37740450c51919e3809472f79060f4d64cf0/pydantic_core-2.47.0-cp314-cp314t-win_arm64.whl", hash = "sha256:25cac08c9735e61e5c0b9f7a85c438661fc0e9da226afdfa984c3da6c5942a5a", size = 2025906 }, - { url = "https://files.pythonhosted.org/packages/1e/a3/7391a9a511aba60ba119dcbc221a36109dd05ff498dcd6e6ff7222a21b76/pydantic_core-2.47.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:dbe50ffd50c7e1b8e3bedafafd10fabe8a7cf51916f995fd57316eb1293f2439", size = 2112946 }, - { url = "https://files.pythonhosted.org/packages/71/74/1d0fe337fa5a89069731206be4da2b41793b8f8fe28f34ed4d941028a62f/pydantic_core-2.47.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:67c3f4b57a1cda846a9b6258e0ef70f99b0757d0b6a55f7e3e5b100620937d2c", size = 1940704 }, - { url = "https://files.pythonhosted.org/packages/ac/75/d40231aed211f97ffc3264d7a7c98d1f52f7b9c8d165cee75c8c1c13e5c3/pydantic_core-2.47.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33c1e025c83b3987784191ec5aa78cfb94686f1ba73d98ae4d520c09cdecda8d", size = 1985024 }, - { url = "https://files.pythonhosted.org/packages/a6/bc/e27d3fb897c5b059a508b88eb7c4cc1f0fe600be3231728bb003e1e2b464/pydantic_core-2.47.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c547cac87585ab8a6e9495fd3be22ae8ccaabb3c909f6ae589a180677c105cc8", size = 2136202 }, - { url = "https://files.pythonhosted.org/packages/4e/21/81cc7e3acbf7d4eed41e9585e81b5840aaf6ddbe65974a20946038d6516b/pydantic_core-2.47.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:d648515c65f8871ceca6a0446be32fb1a0aae414db3907ec0df6cc2380dd0c04", size = 2098016 }, - { url = "https://files.pythonhosted.org/packages/4f/6e/c98e869e9f754f18088ad9b55887972d08606e7bf81dba088779e927eb65/pydantic_core-2.47.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3292813ac8ed17671a5b1ac8256b6e98b96c29c1c6d061c35899e2f6e0444cac", size = 1931175 }, - { url = "https://files.pythonhosted.org/packages/5d/65/ce0d773f4b51dc332764ba1bd585c81f62d5c92231a66d152aff16be8dfa/pydantic_core-2.47.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf3cfcea028f41a9b42a47f4a53d72ca4274d04e3ee525bd58ca89ea8c5e1910", size = 1995356 }, - { url = "https://files.pythonhosted.org/packages/19/2c/6e63683a77d342c93bad67bb0762ca8fe1d8418acc8617d70ccdcc5ec5ff/pydantic_core-2.47.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52afdd8640d59890539060e91c8a494cf96b463e63b2ba499cc5c23abcb5e96b", size = 2144893 }, - { url = "https://files.pythonhosted.org/packages/f1/bf/06a01e2f6668aa10646e5c9b050a609d975a93d58454e26e16fd9f82cd38/pydantic_core-2.47.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ff201e8b93cd6a3849ec3fecb1d642d0391ca4a387f1162adaadddfd1986598c", size = 2114751 }, - { url = "https://files.pythonhosted.org/packages/41/50/caf4cc67fdd664baf89bfe264d72079e901a02d903c813ced9de173cdd69/pydantic_core-2.47.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eba39fca6e1fc7c58a5e6b5c320c6eec86014f7075dfa0fb1a79076e00304b3a", size = 1947894 }, - { url = "https://files.pythonhosted.org/packages/9f/c0/96ba102865c9ccb945a758272741d00175837bf32bb71750f6e2bf4543fb/pydantic_core-2.47.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b494566e8e7e1a3992a550900902d6c0051538e53f5957e24003e7775638a24", size = 2131698 }, - { url = "https://files.pythonhosted.org/packages/b9/f8/ee7c5d4eafaecdeaf92ce84f48d748c139888e90fa25b2ae988c5aea874c/pydantic_core-2.47.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c96f745745fe778a92e74f40675c57b80dd46fc98186c9020e5ba4514a0470bd", size = 2157797 }, - { url = "https://files.pythonhosted.org/packages/08/69/b3eba1d0e015cee3906f7056cfd9ed383f6cef486fde5398a002df2594e4/pydantic_core-2.47.0-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dbfb8f8ac48a8fd5a9f83cdbba0a23796f3ff5f9421c735808ed134d3318046f", size = 2177829 }, - { url = "https://files.pythonhosted.org/packages/8a/36/a0ceff9b7c3a0980128c9ef763db1e9fe9c2b1fcecf6ed055e88af39f123/pydantic_core-2.47.0-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:4ded2d8dc3beb086623ab3256b81a0b51fc026c94eb363f480aec09b204a1cbd", size = 2307425 }, - { url = "https://files.pythonhosted.org/packages/d6/0a/930e210e95ade33ecacabc4ba822aa1662a0aaf843d2cf57abce27fb95b1/pydantic_core-2.47.0-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:25f8aa59fd3f85651387c37b289cd8f0bbc8f632e7dad13c6b837759e78be88d", size = 2351547 }, - { url = "https://files.pythonhosted.org/packages/2d/0c/90941408bb6c1e84563d99177c9e62c536dce0388b5a8bf37b90b4f6f095/pydantic_core-2.47.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:0ae0096729c64aa6155cef74e75d1945017b01cfb82f313a18d843fafc497476", size = 2180871 }, +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872 }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255 }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827 }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051 }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314 }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146 }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685 }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420 }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122 }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573 }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139 }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433 }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513 }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114 }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298 }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158 }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724 }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742 }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418 }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274 }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940 }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516 }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854 }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306 }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044 }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133 }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464 }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823 }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919 }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604 }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306 }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906 }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802 }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446 }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757 }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275 }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467 }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417 }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782 }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782 }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334 }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986 }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693 }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819 }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411 }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079 }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179 }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926 }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785 }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733 }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534 }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732 }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627 }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141 }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325 }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990 }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978 }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354 }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238 }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251 }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593 }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226 }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605 }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777 }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641 }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404 }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219 }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594 }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542 }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146 }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309 }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736 }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575 }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624 }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325 }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589 }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552 }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984 }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417 }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527 }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024 }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696 }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590 }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782 }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146 }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492 }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604 }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828 }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000 }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286 }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071 }, ] [[package]] @@ -3854,27 +3850,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.21" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/36/6f65aa9989acdec45d417192d8f4e7921931d8a6cf87ac74bce3eed98a8e/ruff-0.15.21.tar.gz", hash = "sha256:d0cfc841c572283c36548f82664a54ce6565567f1b0d5b4cf2caac693d8b7500", size = 4769401 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/c6/ede15cac6839f3dbce52565c8f5164a8210e669c7bc4decb03e5bdf47d0d/ruff-0.15.21-py3-none-linux_armv6l.whl", hash = "sha256:63ea0e965e5d73c90e95b2434beeafc70820536717f561b32ab6e777cb9bdf5d", size = 10854342 }, - { url = "https://files.pythonhosted.org/packages/28/9d/d825b07ee7ea9e2d61df92a860033c94e06e7300d50a1c2653aac27d24fe/ruff-0.15.21-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0f212c5d7d54c01bbfe6dcab02b724a39300f3e34ed7acbe995ccb320a2c58bd", size = 11139539 }, - { url = "https://files.pythonhosted.org/packages/f5/de/3b107712e642f063c7a9e0887c427b22cb44097de5aab36c05f2e280670c/ruff-0.15.21-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e6312e41bc96791299614995ea3a977c5857c3b5662b1ecef6755b02b87cb646", size = 10595437 }, - { url = "https://files.pythonhosted.org/packages/9a/6f/b4523cc90ba239ede441447a19d0c968846a3012e5a0b0c5b62831a3d5e3/ruff-0.15.21-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01d65b4831c6b2a4ba8ee6faa84049d44d982b7a706e622c4094c509e51673be", size = 10990053 }, - { url = "https://files.pythonhosted.org/packages/92/cc/c6a9872a5375f0628875481cf2f66b13d7d865bf3ca2e57f91c7e762d976/ruff-0.15.21-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2c5a913a589120ce67933d5d05fd6ddbcc2481c6a054980ee767f7414c72b4fd", size = 10666096 }, - { url = "https://files.pythonhosted.org/packages/ab/97/c621f7a17e097f1790fa3af6374138823b330b2d03fc38337945daca212c/ruff-0.15.21-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef04b681d02ad4dc9620f00f83ac5c22f652d0e9a9cfe431d219b16ad5ccc41", size = 11537011 }, - { url = "https://files.pythonhosted.org/packages/ea/51/d928727e476e25ccc57c6f449ffd80241a651a973ad949d39cfb2a771d28/ruff-0.15.21-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16d090c0740916594157e75b80d666eab8e78083b39b3b0e1d698f4670a17b86", size = 12347101 }, - { url = "https://files.pythonhosted.org/packages/1e/88/8cd62026802b16018ad06931d87997cf795ba2a6239ab659606c87d96bf0/ruff-0.15.21-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a10e74757dd65004d779b73e2f3c5210156d9980b41224d50d2ebcf1db51e67", size = 11572001 }, - { url = "https://files.pythonhosted.org/packages/b2/97/f63084cf55444fc110e8cb985ebfcc592af47f597d44453d778cb81bc156/ruff-0.15.21-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bab0905d2f29e0d9fbc3c373ed23db0095edaa3f71f1f4f519ec15134d9e85c8", size = 11549239 }, - { url = "https://files.pythonhosted.org/packages/9d/77/f107da4a2874b7715914b03f09ba9c54424de3ff8a1cc5d015d3ee2ce0ac/ruff-0.15.21-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:00eca240af5789fec6fe7df74c088cc1f9644ed83027113468efba7c92b94075", size = 11535340 }, - { url = "https://files.pythonhosted.org/packages/d5/e9/601deb322d3303a7bf212b0100ead6f2ee3f6a044d89c30f2f92bf83c731/ruff-0.15.21-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:262ab31557a75141325e32d3357f3597645a7f084e732b6b054dde428ecd9341", size = 10964048 }, - { url = "https://files.pythonhosted.org/packages/ea/2e/0f2176d1e99c15192caea19c8c3a0a955246b4cb4de795042eeb616345cd/ruff-0.15.21-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:659c4e7a4212f83306045ec7c5e5a356d16d9a6ef4ae0c7a4d872914fc655d9d", size = 10667055 }, - { url = "https://files.pythonhosted.org/packages/48/60/abd74a02e0c4214f12a68becfd30af7165cfdcb0e661ecdc60bbb949c09a/ruff-0.15.21-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9e866eab611a5f959d36df2d10e446973a3610bc42b0c15b31dc27977d59c233", size = 11242043 }, - { url = "https://files.pythonhosted.org/packages/b2/c6/583075d8ccabb4b229345edcaf1545eb3d8d6be90f686a479d7e94088bbf/ruff-0.15.21-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e89bc93c0d3803ba870b55c29671bad9dc6d94bb1eb181b056b52eb05b52854f", size = 11648064 }, - { url = "https://files.pythonhosted.org/packages/3a/3c/37d0ecb729a7cc2d393ea7dce316fc585680f35d93b8d62139d7d0a3700c/ruff-0.15.21-py3-none-win32.whl", hash = "sha256:01f8d5be84823c172b389e123174f781f9daf86d6c58719d603f941932195cdd", size = 10896555 }, - { url = "https://files.pythonhosted.org/packages/c0/b8/e43466b2a6067ce91e669068f6e28d6c719a920f014b070d5c8731725de3/ruff-0.15.21-py3-none-win_amd64.whl", hash = "sha256:d4b8d9a2f0f12b816b50447f6eccb9f4bb01a6b82c86b50fb3b5354b458dc6d3", size = 12038772 }, - { url = "https://files.pythonhosted.org/packages/dd/75/e90ab9aeece218a9fc5a5bc3ec97d0ee6bb3c4ff95869463c1de58e29a1c/ruff-0.15.21-py3-none-win_arm64.whl", hash = "sha256:6e83115d4b9377c1cbc13abf0e051f069fab0ef815ea0504a8a008cee24dd0a8", size = 11375265 }, +version = "0.15.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/06/ae069393fc66e8ff33036d4b368003833bf6e88ccf182e17e7a2f1c754fd/ruff-0.15.22.tar.gz", hash = "sha256:3f15175b1fb580126f58285a5dae6b2ea89000136d980c64499211f116b54809", size = 4785063 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/18/ee54b7ae1e121be7a28ea6da4b67564ebb0530e183a54415ab7e3bcd2c4e/ruff-0.15.22-py3-none-linux_armv6l.whl", hash = "sha256:44423e73493737f5e7c5b41d475483898ff37afcdae38bc3da5085e29af1c2d8", size = 10781258 }, + { url = "https://files.pythonhosted.org/packages/2f/d2/2520cb14761ddbeaf57642a76942fc36adcbdbe53b4532241995f6fc485c/ruff-0.15.22-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b82c6482946e9eda7ff2e091d25b8bad3f718684e1916d41bd56873cee05b697", size = 10999477 }, + { url = "https://files.pythonhosted.org/packages/c9/10/74e53572aa758dfaa678c2a2646b5c5515d884b7ca56be4d2ce03ca4b560/ruff-0.15.22-py3-none-macosx_11_0_arm64.whl", hash = "sha256:11c1c715af53a09f714e011106bffc419751ec8232fcb5da42173284ea3fec6f", size = 10466716 }, + { url = "https://files.pythonhosted.org/packages/1e/cc/44eaaf0844e028182f2d0a8f2190d0f359159aed0a9e5ab861d892f1ae2a/ruff-0.15.22-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:742a29cf29bddb7c8327895d6a10e0e6c5b38a96dd407af9b5d0857f809c0576", size = 10892644 }, + { url = "https://files.pythonhosted.org/packages/9f/21/8edf559014d2b0f82beea19cfb713993ad802ccda16868769979c6090a84/ruff-0.15.22-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72af58b951b0ae395935ae79763dc349bc0eb706319d28f7a33ad2cfb3cfc178", size = 10576719 }, + { url = "https://files.pythonhosted.org/packages/bf/1e/3a13abd392a3b50b62e5938a831f9ab6e588358cacad5c18545b716d2182/ruff-0.15.22-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62d425005c1835eb24e2ee4161cb90e8db263415f4a71c8c72c33abaa6c0c224", size = 11376494 }, + { url = "https://files.pythonhosted.org/packages/bf/3e/422d3d95bcf04dd78e1aeac22184d4f9a8fb2c01865d39d44618484a0317/ruff-0.15.22-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8b9b3f8779a4f08c969defc3c8c35abffaa757e601ed5ae66d6d1db6519969a", size = 12208370 }, + { url = "https://files.pythonhosted.org/packages/1e/91/5d065a0e0a02bf4813f5119ad278462eed081d2b832eb7c021ade0ec9e65/ruff-0.15.22-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e0dd1b2e4d3d585f897a0d137cbf4eaf6223bef4e8ce34d6bb12556c5f9249e", size = 11581098 }, + { url = "https://files.pythonhosted.org/packages/f6/f9/a0d4871d12fae702eb1f41b686caf05f1f8b124dc6db6f784f53d74918fa/ruff-0.15.22-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:365523eb91d9224e1bcb03b022fbf0facb8f9e23792a2c53d9d4b3924bdbdebb", size = 11399422 }, + { url = "https://files.pythonhosted.org/packages/18/80/c843a5176cddbceb0b7e8dd41cf9993490796c1c469348d384f5a5c13c56/ruff-0.15.22-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:fabfd168afdf29fee5be98b831efa9683c94d7c5a3b58b9ce5a2e38444589a74", size = 11381683 }, + { url = "https://files.pythonhosted.org/packages/d4/00/8485de0ae92239438a36cfc51350db9b9e85c9ebdfaea91b18e422706662/ruff-0.15.22-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:225dbf095a87f1d9f90f5fd7924d2613ee452a75a4308c63a8f50f761787aa7c", size = 10850295 }, + { url = "https://files.pythonhosted.org/packages/fa/91/24977ec2ec72eaf15e4394ace2959fdff2dd1e14f03e005e838023407169/ruff-0.15.22-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1877d63b9d24ed278744f1523fd11b85540566d54641f97c566d7d9dc5ca5296", size = 10579640 }, + { url = "https://files.pythonhosted.org/packages/9c/47/9b51216951974df1f263ac19da550d34252e0ed7218c25f10c5ef9ed7517/ruff-0.15.22-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a1606c510bd7215680d32efab38965f7cdec3ef69f5170a3f4791404ffdd5262", size = 11105077 }, + { url = "https://files.pythonhosted.org/packages/c2/47/20e9d4a3b8016778acea5fc32bb50d35d207500a17ddb529ffa6996feef8/ruff-0.15.22-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:630479b18625f5ffc373f77603a22a9f8ac0acd7ff0501178b5db28ec71e9c64", size = 11490980 }, + { url = "https://files.pythonhosted.org/packages/4d/76/3f72d8fc38c1cb77b38c56a70da9d0c17700cc1cc50f9649c9d3c8f5ba71/ruff-0.15.22-py3-none-win32.whl", hash = "sha256:e5ba0e4a13fd14abbed2a77b517a3911290c6c6c59ef67784328d1668fab76cf", size = 10789165 }, + { url = "https://files.pythonhosted.org/packages/cb/46/4965251734c2b6fcdca1b1b187d20bcac3af0ee5b083b89c910bb961ce3a/ruff-0.15.22-py3-none-win_amd64.whl", hash = "sha256:9be63ba1eb936acd2d1342fb8337c356353706fce233b2a15a09a97037e6acde", size = 11938297 }, + { url = "https://files.pythonhosted.org/packages/57/c9/e69b1ff4c8b69093ef08b8919ab767af0569666865b39c30a8795d88d3c6/ruff-0.15.22-py3-none-win_arm64.whl", hash = "sha256:e1168075b72158510839f250027659cdd78476f40507dd517892304c41318661", size = 11298172 }, ] [[package]] @@ -4100,14 +4096,14 @@ wheels = [ [[package]] name = "smart-open" -version = "8.0.0" +version = "8.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/29/3e/79fd5fd2375a8a500b9ec2f6a0762fc1ac33e35582d4a87483a78d19408f/smart_open-8.0.0.tar.gz", hash = "sha256:5a2008d60688bd3b33c52e2ef666d3c60cf956e73e215de8c7b242cf56fdd1b2", size = 61520 } +sdist = { url = "https://files.pythonhosted.org/packages/76/53/9c513747547fd595d5c143259129ea8b9c3ea2f6b7bb9dcea2b1966ded3c/smart_open-8.0.1.tar.gz", hash = "sha256:18b1c4496003c6902be17c15f032b5c319f307c89c6ae9e6b028b508bed8b2cf", size = 61882 } wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/bd/1c92e69a1daff70118f21e18ef3a100c114f00f08b64a1074484f12d9020/smart_open-8.0.0-py3-none-any.whl", hash = "sha256:ff4f395c9e86f23e27771dc4ba756ad4bd145f181859a782c50d64168485761b", size = 73029 }, + { url = "https://files.pythonhosted.org/packages/c3/96/325b8c507ccecc50421fecc0345a502ee6e4a44785af3c4e6ecbadad624a/smart_open-8.0.1-py3-none-any.whl", hash = "sha256:3e97f90e92a952cb57863dfe132082c400a52eeeb27c067692fb51dbcc5b0089", size = 73504 }, ] [[package]] @@ -4562,20 +4558,20 @@ wheels = [ [[package]] name = "trimesh" -version = "5.0.0rc1" +version = "4.12.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4f/73/1c56a0e3a91dcb36d0957b2d87087cd3300e4b341ee0eb1711f71760d589/trimesh-5.0.0rc1.tar.gz", hash = "sha256:2f48db2cb769ad379dadbee16fcf8c36b4814c72bf237f3c5941076abc829e61", size = 849395 } +sdist = { url = "https://files.pythonhosted.org/packages/79/37/5cb90f04990260d2caceb6093560c6cefafca1ec522c1e43be01ca658244/trimesh-4.12.2.tar.gz", hash = "sha256:c8ca31571ac00b112e4e160e66a2d4c3491df321f056bd33806be0485d1af9d9", size = 842220 } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a7/5d3fc0b3da684a9984438e8ba00d8d5528013ba32836894bb18875590d5b/trimesh-5.0.0rc1-py3-none-any.whl", hash = "sha256:2ebd562ede63ab2089ab751f5fd151758c5009aa903dfd18add5cdcfd7cfcc36", size = 742125 }, + { url = "https://files.pythonhosted.org/packages/05/98/716a473cfb24750858ddd5d14e6527539dd206583a46408d08eeb2844a75/trimesh-4.12.2-py3-none-any.whl", hash = "sha256:b5b5afa63c5272345f2858f7676bc8c217dc8a89f4fadf6193fe10a81b5ff2aa", size = 741043 }, ] [[package]] name = "typer" -version = "0.26.8" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -4583,9 +4579,9 @@ dependencies = [ { name = "rich" }, { name = "shellingham" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7c/f7/68adc395201b20b872d68e975386832e8005ffeacedd43a1d837a32815be/typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e", size = 202097 } +sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430 } wheels = [ - { url = "https://files.pythonhosted.org/packages/80/87/b9fd69c92c6102a066e1b86a35243f53e70bd4c709f2a26d9f4fee4f4dc0/typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c", size = 122564 }, + { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716 }, ] [[package]] @@ -4742,77 +4738,77 @@ wheels = [ [[package]] name = "wrapt" -version = "2.3.0rc2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/d1/370cd471ee21bfd077a3c32e25c60366e0dc5c2df475eab98dbfa5118b3b/wrapt-2.3.0rc2.tar.gz", hash = "sha256:1fdc5475357ab9bb7131f2d70f72f55a1a7fb4ecf450b2868a61440e9ca2e9d1", size = 131510 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/98/4e03d97b40a975e6bb03949f0662c3ed54c7886841d3f1fd98293ed9d4d4/wrapt-2.3.0rc2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a1b06c694b4b3d34211e24cff1f56e8aa500840d6c40ad6970055467d58a916", size = 81479 }, - { url = "https://files.pythonhosted.org/packages/a7/c4/d802f7032af83b26e2d4d92149a5823b5e96285da9978bd0292fbdf4d83e/wrapt-2.3.0rc2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a435634eea5c06ee30478dd277af4a0037171a53e2cf3269a748eee36ba24009", size = 82412 }, - { url = "https://files.pythonhosted.org/packages/14/59/a1a54f8d54ee4efbf1fe151843a013dbda3e28a7c136a5786399190506da/wrapt-2.3.0rc2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cdfa341cfd63118dfc83eb43125fc9a12e0dd5293e159a8b8b0202ee72d0f07", size = 161750 }, - { url = "https://files.pythonhosted.org/packages/b9/75/59d9a2f6a2037c170aedd86075ba214016a993a2f84b6ea10dae0eaf822a/wrapt-2.3.0rc2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b6c23b614246ae9322f259e4d99e5e90dc69fdbc5e2b81fc4bad69b81ad9a54f", size = 162971 }, - { url = "https://files.pythonhosted.org/packages/c8/1a/ff83561d97814af0e39a7924233a5cc4d5874ffba135bad1923a8365bd7a/wrapt-2.3.0rc2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f92e33fd8f74f1ad4dac902d96e74a15cf43c4dfd0466f5530bdc93b303d5be", size = 156174 }, - { url = "https://files.pythonhosted.org/packages/55/48/6b16298fbd1643d6363bc8f0e70558f6d62f93980f868725274e5ef644f4/wrapt-2.3.0rc2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:356e07162c524014735e708f3046390ea775af076fb22bdc21c2b1dff3030f43", size = 162088 }, - { url = "https://files.pythonhosted.org/packages/78/82/7570326e6a58292209c55a39b0def5f7a5a49090198a1ba3a46b5a201432/wrapt-2.3.0rc2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:cc33796b0542f9ab58b12315f5619a19429230921bff0cd3cbbbb8f91f1f7958", size = 155160 }, - { url = "https://files.pythonhosted.org/packages/7e/ac/c907e7549a6baac70576dfdf7836444e3be630e13087b320f8de817bb348/wrapt-2.3.0rc2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6a93960a07dcdc8f543f3d4d12cf2c54c5c212d1b082d837bac3b6b7171fa64", size = 161140 }, - { url = "https://files.pythonhosted.org/packages/3f/b8/637e499f762850af9976594c643797e008634575f4523c63791653bc4b1d/wrapt-2.3.0rc2-cp311-cp311-win32.whl", hash = "sha256:ba257f67b3988cdcab3c142678dace39f4ece395753b4b5af787afdc811fc258", size = 78072 }, - { url = "https://files.pythonhosted.org/packages/52/80/90566e8ecc181af896bb41c8839bed2a066faf0a80d9c3689a983fb95d6c/wrapt-2.3.0rc2-cp311-cp311-win_amd64.whl", hash = "sha256:adc98263049b268dbc81a40713e472fe1a9b3bc2a19c25e4416e393a3cbbfa08", size = 80986 }, - { url = "https://files.pythonhosted.org/packages/cb/3b/88ccd09ca3fe40feb1db0ea42dcd4fdee90ccc1b9d352b8226454d551950/wrapt-2.3.0rc2-cp311-cp311-win_arm64.whl", hash = "sha256:60ffdecfcbeaa2cecef267ed5a88c2589d62d19347dc8c2d9cc72a29ff5bd0a0", size = 80118 }, - { url = "https://files.pythonhosted.org/packages/e6/49/fe1c897a1a51c7ff7a70c20ab532945fb663e3c23ea5adbad5c49465a2dc/wrapt-2.3.0rc2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9fd8e98222d10650038117a4d44bba463ebfdf341dd0e23434da82364f44d91b", size = 82191 }, - { url = "https://files.pythonhosted.org/packages/2d/1b/67dc64e9f97b6e6103d80409dfa105b8a40d2bb87feacd0e06da392fb75a/wrapt-2.3.0rc2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aa545b5865ab879725eb61b7170be079901577a16996a01825e2b8617ad217aa", size = 82777 }, - { url = "https://files.pythonhosted.org/packages/9e/fe/9743f3f9d74e0a9c5c5719563895e9df58c8646c95c956cf4fab667b9290/wrapt-2.3.0rc2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1f50df40e79f7fc09454dc6d562fad76c7ac6d2f93359f61e2155530f899cb0b", size = 172434 }, - { url = "https://files.pythonhosted.org/packages/dc/50/976976c3c99cee28a0b63d08406b83e49189c6a430adbbc202c20d900e2a/wrapt-2.3.0rc2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87b45ac248af8fa977f2a8b86826edbf63badcbdbc76f20d87af2edbadd7512d", size = 174169 }, - { url = "https://files.pythonhosted.org/packages/6e/e9/a1b3548c82b90f8ed190b3c05b3c9995c4260ac0f0a644c6002c584ce980/wrapt-2.3.0rc2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f2583f0906dfbcf55f44db7a5a90f7471b88924ae99f414c9c2d64c8bd8156d", size = 163086 }, - { url = "https://files.pythonhosted.org/packages/16/44/8d27984fae74bb8ebbaa6b06157cf94c97ef0ce764e2497c21ce4329bc9e/wrapt-2.3.0rc2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f65a06464eb6f932daf82c275e76b6e9406fe1025cdc31ece5dbc310da37ab98", size = 171939 }, - { url = "https://files.pythonhosted.org/packages/b8/b0/e1ed933e871d9a76b41c8112bb5bbefc035aab1bccce50bd855d52fed78a/wrapt-2.3.0rc2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a70051c361b763a8389b89c2cbd1c5f555924ed01e5a113bb0554ac9cc7377e9", size = 161163 }, - { url = "https://files.pythonhosted.org/packages/d8/1f/97c7a3e0cd0df429ab75d6cabdfb2a76fa992e0614f15517ebb4a6fbaa73/wrapt-2.3.0rc2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7e6595f3a161b056750f558ba02749fa0c3c3cfb410c9cc0cce756232a38f388", size = 170582 }, - { url = "https://files.pythonhosted.org/packages/d1/71/04af17fcaded6c659fe00153bcc4de0c93cadd36c9986b4438b08c9162ed/wrapt-2.3.0rc2-cp312-cp312-win32.whl", hash = "sha256:ef50260aeb9585d6ce4d4d5f9fdb33d3933f5768feeb82e31971d97823a393aa", size = 78363 }, - { url = "https://files.pythonhosted.org/packages/f7/a7/78b0a679569a93bb4b933e46a8ff03d3acdb9548e0ccc1fa0251920726f7/wrapt-2.3.0rc2-cp312-cp312-win_amd64.whl", hash = "sha256:6034972419dfc7e73dc8d901dd47b6ae99a33b270c1c3736f59927c78d4c8e2f", size = 81222 }, - { url = "https://files.pythonhosted.org/packages/70/c6/f1fd8ecb62fd5661a45514fab46d24fbed5afc25dd11368b45c928c811da/wrapt-2.3.0rc2-cp312-cp312-win_arm64.whl", hash = "sha256:ccfd86c2abc39e1629c5f1423fade1c6a930442872b0edd65015465413e1e235", size = 80196 }, - { url = "https://files.pythonhosted.org/packages/68/2f/dd1cf107c37f7201e7cc23792777978c9f5de7f15ed235d94c4a49879657/wrapt-2.3.0rc2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:49d51c281ced9a2f185474e24fb9c73a36339a22db2fd9813a5617758a184c05", size = 82013 }, - { url = "https://files.pythonhosted.org/packages/67/fa/92b1d74005d8c22a80820d0f8eae0df81fc792beb7aef4fed4c4b1754c41/wrapt-2.3.0rc2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6793d176dd48e6131c4468a31f4b28beba674b22213b5f2807d61890c95dc131", size = 82488 }, - { url = "https://files.pythonhosted.org/packages/80/5f/615db0b23e0e7a6adc206e2e87838d049718f7bad558a30ecd1ebc98ee8b/wrapt-2.3.0rc2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb587f862a866a0eb3f57c7b302dbdcaf27148022c9442434e283db3931bbc88", size = 170401 }, - { url = "https://files.pythonhosted.org/packages/8e/6b/96f238949b228b086a3b21777b3b73a94113480d39f8c8c7a02333c88b45/wrapt-2.3.0rc2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b240a3e51207aafd3da82479b2916cdc0bf16dc855d47b9cac9ab66ce27ac394", size = 170073 }, - { url = "https://files.pythonhosted.org/packages/83/c9/532c66185dfed70ff8c080489674de44d93e3c0027a18a3b55b26b9e9bab/wrapt-2.3.0rc2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d08e69852a0415aa7d938f208b069e3f9bcf214365c718561bf80ec9db7792a", size = 161093 }, - { url = "https://files.pythonhosted.org/packages/c0/0f/0c428cfafdb1f507ae34d5f8c741260be4b8eecf0c44bf3bdb1ad1beb38b/wrapt-2.3.0rc2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e9e1e8dea44583f017608c4a07fbc13398d19deb45bf07363100e17802c2fef5", size = 168626 }, - { url = "https://files.pythonhosted.org/packages/2e/05/490933784de4dbfc792c89a5ac5a15ec557f46fc1516fc0b25e0b8258ea9/wrapt-2.3.0rc2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7dce02acb173a60cea89aa143e5fcc38b74d6de33366daa9484246741e9cc6f5", size = 159191 }, - { url = "https://files.pythonhosted.org/packages/9b/0d/1be4469ec4b302de6636e9c7aa2f7d932a7b85a0e89b0ae4f8c103496fe5/wrapt-2.3.0rc2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:62da2c71924d7772fe8a924f84be82731e7f1546169dc278be18cf1a2cf726f1", size = 169313 }, - { url = "https://files.pythonhosted.org/packages/42/c1/21f763de3be84b33a058999ada940f0724ff6d10078df728745c354992ea/wrapt-2.3.0rc2-cp313-cp313-win32.whl", hash = "sha256:dc742c92a464ab3e24f7ee1e472861c51a3c3e5e038f08e9a7720b24f9737525", size = 78284 }, - { url = "https://files.pythonhosted.org/packages/b0/ab/6f860a0ce2411e805d373f3ae501739dc50a462239c08e27dbeded7a684c/wrapt-2.3.0rc2-cp313-cp313-win_amd64.whl", hash = "sha256:172ce1c5c97241fe0ea8a73bc2d23b5508986301cf71623b7c87a73510339e33", size = 81155 }, - { url = "https://files.pythonhosted.org/packages/72/d7/142a0166da98a2cc1c4bb890214228a1baf91b21760e86f22124707fdaec/wrapt-2.3.0rc2-cp313-cp313-win_arm64.whl", hash = "sha256:ee8464a1ac8c27695fb79500f23549c5fc8dc7f3cec48ffaad99a6972dac31cf", size = 80221 }, - { url = "https://files.pythonhosted.org/packages/c2/42/02a459c2d8629cca66f4e092b4da48eec60fef7e0203a3b0b9930ccb81a3/wrapt-2.3.0rc2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:32d8c5e2fe0941ca5bba1d1daf312d75d2c693d071c5b8c54f0cc674b22dcc45", size = 83975 }, - { url = "https://files.pythonhosted.org/packages/db/42/36999e913db9c3e26019e48dad0a32261e6afe6e5496a88687679e0f8ad4/wrapt-2.3.0rc2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cc4d69d2bb09068bbabe98f40e8535a0d632cc40504e94fafb9837e332f12150", size = 84469 }, - { url = "https://files.pythonhosted.org/packages/21/73/e58138fd45a5c16b9efe0f1c989efa7024d32062451426d76c2f8333b70f/wrapt-2.3.0rc2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c079e72c727cebf21f2df8a847b3486d21833edd18941ce14a13bbf36d6f8970", size = 207220 }, - { url = "https://files.pythonhosted.org/packages/50/00/b8e2649b2a1da6bf9cb6478e22c5a9196fc458f3569e32134f1315cc7572/wrapt-2.3.0rc2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b72686e310c8465caabb0d17a1df618b5c38281e888a58bfe395cc02de743aa", size = 214401 }, - { url = "https://files.pythonhosted.org/packages/b0/a1/149556616e0dc6b57bd740c8a2b866e1c5f5b8f91b8a4919a0f524ebc723/wrapt-2.3.0rc2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7d36027d85e87b80b1525f49c2ea945d946f7f92735d273313bdc86a8d0190e", size = 199075 }, - { url = "https://files.pythonhosted.org/packages/82/28/afeaf4e958646d8be0af9b81e4349747405ce021362bb8406e4ca4192d36/wrapt-2.3.0rc2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:3151b57bd2834364cbe9dd60b41949e9695fd582b949903619ef8637d14baf9a", size = 210021 }, - { url = "https://files.pythonhosted.org/packages/43/b0/061bf4cf9c3aa431b7fab4f6ec11683db5cb2a90e254f5ef71056cab7db4/wrapt-2.3.0rc2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:8a638bafd782f1022c70f4abf1bbe81850379c4e793e88b3d71026bf47043b8c", size = 196375 }, - { url = "https://files.pythonhosted.org/packages/46/76/eadeb5928d548f827e78617f27ee531d0cd3f623fb2a7ac0f8171cfd16ee/wrapt-2.3.0rc2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d15c3cb205758d6fda3465041d47f94e17e660ed94288e242a19e4f7f83c88ae", size = 202661 }, - { url = "https://files.pythonhosted.org/packages/86/32/ea620316f4ecf3198b0f7a7e03ea89ceeab48888986a818d296cfe9cd6ea/wrapt-2.3.0rc2-cp313-cp313t-win32.whl", hash = "sha256:3931aeab99382e388573bc774ee7da46f01601cddf96d82d992c07854c741970", size = 79220 }, - { url = "https://files.pythonhosted.org/packages/bf/27/b6a58c6eaf9cf9ab8aa2082d16961e7eeebe7c1e4c3897fcebe8a321081a/wrapt-2.3.0rc2-cp313-cp313t-win_amd64.whl", hash = "sha256:920899e38a9edf38a778d14af2a8acd5476ed3b91878a1e12766f32944c6f73d", size = 82676 }, - { url = "https://files.pythonhosted.org/packages/03/36/c3bf6a4876fe9c9fad1b5fe031a514651537c2aa7760e975867e18e83ce4/wrapt-2.3.0rc2-cp313-cp313t-win_arm64.whl", hash = "sha256:a7ff861497846dcdb22549a1afcf593fba20c62273555f86cfe0083cf8dd7583", size = 81430 }, - { url = "https://files.pythonhosted.org/packages/42/fa/3dad956730f5a0a8f9e1bc3cd6605504ba78bb94dca210b3efa5ad36324b/wrapt-2.3.0rc2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c2d2b9e1935fcdb95fabca0fc6dae2afa956a9e78853e77f55ce33c3724126c2", size = 82028 }, - { url = "https://files.pythonhosted.org/packages/5d/2e/4bf6483f09f10cb61c767f8d13770b9cca22f58b9b1fc721c91f77a0d6cf/wrapt-2.3.0rc2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d029368d9eef6b6319599996b3f1cec7e954e4b9b7e41db2aeae41e9a18cfb02", size = 82568 }, - { url = "https://files.pythonhosted.org/packages/51/37/6b8c84242e01e02903314ceb9b13be501afb266d75681ddd2b20d8494b6b/wrapt-2.3.0rc2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:70e2dbb96176fdf7726cd105930da2a9ceef835b7d4f938133d1cc78384f0277", size = 170239 }, - { url = "https://files.pythonhosted.org/packages/55/50/70bdb60298ae60f19debe84269fb97381f32537b75ca6dd3118f9c48e8ba/wrapt-2.3.0rc2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b33d2a48bbe38c0a3645dd8a94a8d57476de5333c0a923147e366f757d08b777", size = 169339 }, - { url = "https://files.pythonhosted.org/packages/4b/f2/7631042e0f6d52563b75dc5abd400d1fc4d0f934e9a5026f6c4b38f7dfe8/wrapt-2.3.0rc2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c61ad736febb7da2c29294bf3430da1657aa7f9dec1e832e906520382267beff", size = 160985 }, - { url = "https://files.pythonhosted.org/packages/d7/fc/dd1db1fb05a7355f5a019b5a7a7838f43d9172ba4f68acb386e37ce7250f/wrapt-2.3.0rc2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:20622c6d8bab477f56bf2f7203119906d0fb37ef0b91d8128a9d7be2b62c18e0", size = 169066 }, - { url = "https://files.pythonhosted.org/packages/e6/57/d87205122d6e2c969b86047333a512325e104137d1fafd0b45a9cb0a168e/wrapt-2.3.0rc2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3c9fe3cd6c35ea9b446586c5a955e3778b859eb414abbadf0820cf9b7ac0c69e", size = 159116 }, - { url = "https://files.pythonhosted.org/packages/cf/b8/cde4b0b0e708b9fc9d4f864333c59253b6b1e84bb80f8a03ee7d065b50b7/wrapt-2.3.0rc2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a754ca8bc82f408ca48209a4c49def3e73d8c45bab346be5ce7583d0c3870308", size = 168871 }, - { url = "https://files.pythonhosted.org/packages/98/69/a12b5645a6d3702c8fc690354092be086855d6759bc49e3714af954d9b0b/wrapt-2.3.0rc2-cp314-cp314-win32.whl", hash = "sha256:3112a5bd8fad521637d15503a2e4445a44b2b725fc292d3446c622bbf333255f", size = 78743 }, - { url = "https://files.pythonhosted.org/packages/09/94/2bc3841e399dbabc389fca197510b20a9ee0c6c93c53d699a62592630654/wrapt-2.3.0rc2-cp314-cp314-win_amd64.whl", hash = "sha256:8117375ad079da41a259deaa0d07a9664f6e8b6f25b29be5ced9ada20a961d88", size = 81467 }, - { url = "https://files.pythonhosted.org/packages/da/96/749c961fb33f294a75d09461fdbc53923caf951372fb4f3207eac2992353/wrapt-2.3.0rc2-cp314-cp314-win_arm64.whl", hash = "sha256:2a3c46ab6cfc0c71dbecaa97d2a53b61596a63d846e1c2d049a2171ed98818a2", size = 80683 }, - { url = "https://files.pythonhosted.org/packages/8a/e9/ea89ea9b6ac3364c1b68a31ea6b14ccf009355a91f56a943aa730b5d53c8/wrapt-2.3.0rc2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:277d795ec7e8db68e9003d9d0322dce883014773a5c523ea860c3f4adf939d35", size = 84085 }, - { url = "https://files.pythonhosted.org/packages/02/b5/5f22a60df1dce28d735dd3d3a0bd03c9a66d0db0fd796555952f03db40f8/wrapt-2.3.0rc2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:008b1c296c8459616bff5b5f2e243493513897b89d33d2e37e2e226dc7f2b8d2", size = 84471 }, - { url = "https://files.pythonhosted.org/packages/53/a1/a2f06f90c0ee04d9e9c4222836a1997e547138fb42f8345f861b4561238c/wrapt-2.3.0rc2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:27622f51bb39261193c4d20dab67827c993e897430215adc54776e882fb09081", size = 207222 }, - { url = "https://files.pythonhosted.org/packages/a8/a2/0d1baa2d01e5eb5ce13408fdda2d2bcfb3124bb92b92b87f69316a614a68/wrapt-2.3.0rc2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:91a719739e842051f2deb7d754a125a5a1f17b6fb11a1c942292d47a230cfb23", size = 214380 }, - { url = "https://files.pythonhosted.org/packages/47/70/aab9cf12134b8039f730c15880f8587a4982f3259cc64706ca8bce2c4a2e/wrapt-2.3.0rc2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6027de92ee0802a1045715efe243ef4cdf5e83b3f29d9339bf4133e090a4a90b", size = 199130 }, - { url = "https://files.pythonhosted.org/packages/43/f2/051422f0a99b1114adf4f220a4d905a501633ea2571a4579ebdaeba05dc9/wrapt-2.3.0rc2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a47effb7489c3348839e6b4918893b9222fb70f4af7dc454be7a6a3f4c61e579", size = 210043 }, - { url = "https://files.pythonhosted.org/packages/d8/b0/ee7663f640cb3d3c8e673038d527c56bb03155191dd3c6c564a2b3427f7d/wrapt-2.3.0rc2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:0e4308e6784b280452dccfb6c3cee5d3b244496235c55c5405dc030923ba414a", size = 196384 }, - { url = "https://files.pythonhosted.org/packages/95/bb/45b6d7d83a19170ab18e1fb8a96b9d2044d6fda3d2e3ab8ded96d88d1e66/wrapt-2.3.0rc2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8a1e7211ed6ccc7dd0adcd389c1bb508c0dd2ce881ad3062224dda158a68f527", size = 202696 }, - { url = "https://files.pythonhosted.org/packages/4d/3a/b3f0c6e28116670f6dbcbfd1b63e462b2a4535d622a00d19a20df5385f42/wrapt-2.3.0rc2-cp314-cp314t-win32.whl", hash = "sha256:f7dd630780d6c7dd2d97bb3c2e2a54e282b5e77fde43acf817a89500faa38ef9", size = 79729 }, - { url = "https://files.pythonhosted.org/packages/8a/7d/5af1eee295e042d22902686b1ce1f605584056235744eea489a1c676f752/wrapt-2.3.0rc2-cp314-cp314t-win_amd64.whl", hash = "sha256:f7feff3c69682d7bc7eef72048a0c560616c24048874ec4fd3673e25e0b97462", size = 83146 }, - { url = "https://files.pythonhosted.org/packages/21/5b/f8f615c2f036916d48361b65caa261222ebf401599d5f4873065dcf363b0/wrapt-2.3.0rc2-cp314-cp314t-win_arm64.whl", hash = "sha256:35f88fc2b65fa74e943d733ce2cbb224072faed4666221a87582b970bd5edbf4", size = 81815 }, - { url = "https://files.pythonhosted.org/packages/db/0b/31d10370b50f53c884bd089e3566c40968c883feeb32b2fcee871d52be69/wrapt-2.3.0rc2-py3-none-any.whl", hash = "sha256:9b477192c1d0aaaf0956cbe6a377e9d62af87401a70f71d392863be70b1ff834", size = 61907 }, +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/15/0c2d55168707465abfc41f33c0b23d792a5fa9b65c26983606940900a120/wrapt-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f1a2ff355ece6a111ca7a20dc86df6659c9205d3fcee674ca34f2a2854fd4e73", size = 80782 }, + { url = "https://files.pythonhosted.org/packages/7d/b5/5c0b093eb48f8a062ef6267d3cb36e9bb1b88440181f6545a383c60efdf8/wrapt-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55b9a899e6fff5444f229d30aa6e9ac92d2216d9d60f33c771b5d76a760d5f8e", size = 81678 }, + { url = "https://files.pythonhosted.org/packages/34/f3/de70937472dd3e8a4e6811192f9c6075efdffd4a2cd9b4596bf160f89668/wrapt-2.2.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a2d78c363f97d8bd718ee40432c66395685e9e98528ccaa423c3355d1715a26d", size = 159671 }, + { url = "https://files.pythonhosted.org/packages/a5/ec/40aed2330e7f02ecf74386ffcfef9ccb7108c6a430f15b6a252b663b1bed/wrapt-2.2.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d619e1eed9bd4f6ed9f24cd61971aa086fa86505289628d464bcf8a2c2e3f328", size = 160785 }, + { url = "https://files.pythonhosted.org/packages/45/04/aa5309beed5344b00220ae6b3b24055852192656194c27947bee1736306a/wrapt-2.2.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:518b0c5e323511ec56a38894802ddd5e1222626484e68efe63f201854ad788e5", size = 153699 }, + { url = "https://files.pythonhosted.org/packages/01/df/2def7e99d1fe87eea413f95f671924cdddcb08823b1ffd212748dfa6d062/wrapt-2.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4bccea5cdecffa9dd70e343741f0e41e0a16619313d04b72f78bb525162ebcd0", size = 159695 }, + { url = "https://files.pythonhosted.org/packages/c7/f6/a906d01a2ce12157bad2404957b3e2140da354b8a70b2fa48bbf282871c0/wrapt-2.2.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:209112cafd963710a05d199aae431d79a28bc76eb8e6d1bbbb8ad24340722cae", size = 152813 }, + { url = "https://files.pythonhosted.org/packages/02/49/bc0086292d239575b4c08f4cf8a4079fa58abbad58ec23abf84833a283ed/wrapt-2.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5a5290e4bf2f332fc29ce72ffb9a2fff678aaac047e2e9f5f7165cd7792e099", size = 158809 }, + { url = "https://files.pythonhosted.org/packages/55/83/8fbd034de1f3e907edaa18786d5dd8f6932874edee0826c7cecb5cab03a1/wrapt-2.2.2-cp311-cp311-win32.whl", hash = "sha256:5499236ad1dc116012e2a5dd943f3f31af12fce452128e2bbcbd55a7d3d4d14c", size = 77414 }, + { url = "https://files.pythonhosted.org/packages/7e/9c/23695baa331c6de4e874c3d78b8e0bed92e1d2a274e665b29858f6841672/wrapt-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:8636809939152be6ae20a6cef0fed9fe60f411b47847d0426a826884b469e971", size = 80368 }, + { url = "https://files.pythonhosted.org/packages/08/49/40cefc342bf89b234a4490d741290fce781774b831aefb39c25471da96c9/wrapt-2.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:5d0a142f7af07caeb5e5da87493162a7b8efa19ba919e550a746f7446e13fb30", size = 79489 }, + { url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484 }, + { url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151 }, + { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828 }, + { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544 }, + { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663 }, + { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387 }, + { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849 }, + { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147 }, + { url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734 }, + { url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585 }, + { url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553 }, + { url = "https://files.pythonhosted.org/packages/43/fc/f32f4b22c6511173c11d9e541ab4e7d8467a0f1b3455acaf784115d31ff8/wrapt-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e8b648270c613720a202d9a45ebabc33261b22c3a839b115ac5bce8c0bb0d69", size = 81296 }, + { url = "https://files.pythonhosted.org/packages/72/06/4d117d5d77a9344776c0248b24dae3d3dd2f58e5f765fa08cf887072e719/wrapt-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6fb7e94e8fe3e4c3067bb1653a91cce7c5e83acc119fdd41501b1bf74654617", size = 81841 }, + { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882 }, + { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411 }, + { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607 }, + { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367 }, + { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176 }, + { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025 }, + { url = "https://files.pythonhosted.org/packages/1c/69/583ed25291ab53e1ec117135fb1c33425e2f46d2bc8f29c17f7a94cf4274/wrapt-2.2.2-cp313-cp313-win32.whl", hash = "sha256:3c4095803491f6ef72128914c28ec05bbad9758433bb35f6715a3e9c8e46fb2d", size = 77605 }, + { url = "https://files.pythonhosted.org/packages/29/68/e69fc6d06e1523c68e0d00f95c9aed1158ce9908ee41603f7f2eae3d5db6/wrapt-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:2cb07f414fab25dbe6b5c7398e1491423a5c81a6209533639969a6c928d474a4", size = 80508 }, + { url = "https://files.pythonhosted.org/packages/55/21/fe7a393d9e5dc0923bed8f5d857e9dcff210f1fa0888c02cc8f3ffaa55aa/wrapt-2.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:1fc7691f070220215cccb2a20836b9adbaecb8ff22ad47abe63de5f110994fac", size = 79565 }, + { url = "https://files.pythonhosted.org/packages/b6/e5/c120d13bf5091164f68c3c1657e84f16f57e71d978421b626393ac5bd7eb/wrapt-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ec8f83949028366531383603139403cac7a826e4011955813cdd640017845ce5", size = 83264 }, + { url = "https://files.pythonhosted.org/packages/d3/b0/d4a1eb97e0e286625bdf21bc7f702637f9607787ffbbdb5ec14d50c79dbf/wrapt-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b481fb0c40d9fd90a5809911208da700987d373a20a4709dc9e3944af7a6bec", size = 83791 }, + { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399 }, + { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461 }, + { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313 }, + { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116 }, + { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668 }, + { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891 }, + { url = "https://files.pythonhosted.org/packages/59/b7/e47651797c097f75a37e2ce86dcf04048ff576f3a674f7c558df7b5e9622/wrapt-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:25904acb9475f46c24fe0423dbc8fda8cc5fbc282ab3dc6e72e919748c53f4e9", size = 78537 }, + { url = "https://files.pythonhosted.org/packages/d1/6f/9fa5d59fb06d890defb5a8f727ce6a14d2932c8760153f96956628559fee/wrapt-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:305d4c247d61c4115794a169141823c62f719525ddb90b23aa332741c77d2c28", size = 82005 }, + { url = "https://files.pythonhosted.org/packages/15/80/4c7bd9873d1f9f7d138d93556b500469dbe24f42710b877519c2b9eb380d/wrapt-2.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c20279cd1a29800815d7b2d6338b60a6c6e78263f9d6e62e0eda251ba9cae2d0", size = 80762 }, + { url = "https://files.pythonhosted.org/packages/24/05/7fd9c3f83b2c74cbfc572a0b88aa37431e04bd8aed70d2c0efd3464206de/wrapt-2.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0e64826f920c42d9d9f87e8cc09ffae66c51ede12d59061a5a426deb9aa71745", size = 81341 }, + { url = "https://files.pythonhosted.org/packages/4b/68/1bfa43100dd90d4ef74a05897b86275cf57e1313ca14aae2545bc9f872c9/wrapt-2.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcaa5e1451bd8751d7bd1568dfa3321c78092a52a7ecb5d1a0f18a5791e1fd00", size = 81921 }, + { url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713 }, + { url = "https://files.pythonhosted.org/packages/4d/9a/d1bd36f6d088c8e652a9383cabbd49af30b8c576302a7eccddbab6963e3f/wrapt-2.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4bfd8d1eb438153eff8b8cfe87f032ba65731e1ce06138b5090f745a33f6f95", size = 166779 }, + { url = "https://files.pythonhosted.org/packages/4c/ae/24ffacd4187fac2740a1972093929e836dea092d42c87d728cd98fee11a6/wrapt-2.2.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c427c9d06d859848a69f0d928fe28b5c33a941b2265d10a0e1f15cd244f1ee33", size = 158407 }, + { url = "https://files.pythonhosted.org/packages/a3/ed/974427668249a356051e8d67d47fa54ef6c777f0fcf3bae9d292c047d4b6/wrapt-2.2.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4250b43d1a129d947e083c4dc6baf333c9bb34edd26f912d5b0457841fc858ab", size = 166594 }, + { url = "https://files.pythonhosted.org/packages/fb/5f/e1d7c6e4523f78db2fbd7826babd0348da1d5e0834c4f918b9ab5757dfae/wrapt-2.2.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:173e5bb5ca350a6e0abab60b7ec7cdd7992a814cb14b4de670a28f067f105663", size = 157068 }, + { url = "https://files.pythonhosted.org/packages/1e/c1/7ebd1027f00700c0b0233b20aceef2b4784294ed64971424c4a78e069e34/wrapt-2.2.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa14b01804bce36c6d63d7b6a4f55df390f29f8648cc13a1f40b166f4d54680d", size = 166470 }, + { url = "https://files.pythonhosted.org/packages/99/eb/974e471a6a978b8180186b8a9dc5ae3361ce269a967190b709b8ce17abfb/wrapt-2.2.2-cp314-cp314-win32.whl", hash = "sha256:58f9f8d637c9a6e245c6ef5b109b67ec187d2faed23d1405656b51d96e0a5b56", size = 78062 }, + { url = "https://files.pythonhosted.org/packages/49/ec/e1281156cdc7a66693838ad7a0865ad641c74abd337a957d668b575aaffb/wrapt-2.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:385cb1866f20479e83299af585375bfa0a4b0c6c9907a981483ea782ea8ae406", size = 80832 }, + { url = "https://files.pythonhosted.org/packages/45/7d/1b6b5ddd94005a2dac97a4490c9838f3154977850d633abcb65b30089437/wrapt-2.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:8ffbeaea6771a6eba6e6eeb09767864995726bc8240bb54baf88a9bb1db34d5c", size = 80029 }, + { url = "https://files.pythonhosted.org/packages/b0/33/9ebcf8aafe91c601127cbd93708c16aa8f688f34a10bf004046803ecdc4f/wrapt-2.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:09f811d43f6f33ec7515f0be76b159569f4057ab54d3e079c3204dddb90afa2a", size = 83357 }, + { url = "https://files.pythonhosted.org/packages/39/38/ec45b635153327b52e52732a0ea980e5f00b7efba65f9e018828f1e69daa/wrapt-2.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a795d3c06e5fbf9ea2f13196180b77aeab1b4685917256ee0d014cc163d90063", size = 83794 }, + { url = "https://files.pythonhosted.org/packages/4e/ea/1a89e6d3b7a83c3affe5c09cde77792c947e63e4bc85ad84cd5bb9abb0d8/wrapt-2.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45c2f2768e790c9f8db90f239ef23a2af8e7570f25a35619ef902df4a738447f", size = 203362 }, + { url = "https://files.pythonhosted.org/packages/19/d8/3b58763d9863b5a73771c0d97110f9595d248db454009e07e1535ee905a4/wrapt-2.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbf00ee0cb55ec24e2b0995a71942b85b21a066db8f3f46e1dbfdb9433ffba81", size = 210449 }, + { url = "https://files.pythonhosted.org/packages/2d/6f/17fd9e053103d8be148d20d5d7505facc72d5fe1f9127973904ceaed79cf/wrapt-2.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2252f77663651b89255895f58cc6ac08fcb206d4371813e5af61bb62d4f7689c", size = 195349 }, + { url = "https://files.pythonhosted.org/packages/ef/04/d0d1ccaaa12cb7dccf28a23f0279a608ba498f71e81d949d5ed54bcfd5c1/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cd7181ab1c31192ff5219269830744b5a62020b3a6d433588c4f1c95b8f8bff", size = 206099 }, + { url = "https://files.pythonhosted.org/packages/44/b3/e8aa07b619890a2aa6cde1931b1887abb08820721b564a5f80b7ca3f3aa0/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6fe35fd51b74867d8b80174c277bd6bbf6a73e443f908129dc531c4b688a20d5", size = 192728 }, + { url = "https://files.pythonhosted.org/packages/b7/f0/1819fb50f0d3c9bd758d8a83b56f1b470dee8b5b8eac8702b7c137cea9d4/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11d95fc2fbad3163596c39d440e6f21ca9fccece74b56e30a37ac2fca786a07c", size = 198842 }, + { url = "https://files.pythonhosted.org/packages/67/7c/e88313f16a99930b899ef970d91c281544a470749a359decad994483bbda/wrapt-2.2.2-cp314-cp314t-win32.whl", hash = "sha256:d8a15813215f33fa83667bfc978b300e35669ea8bb424e970a1426bcb7bc6cca", size = 79059 }, + { url = "https://files.pythonhosted.org/packages/a0/4f/ac12fda57a55068a094ec42851fb0a40e8489d8941863d517452de62e507/wrapt-2.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d09db0f7e8357060d3c38fc22a018aba683a796bf184360fd1a58f6fc180dc77", size = 82462 }, + { url = "https://files.pythonhosted.org/packages/48/a7/df732dac86d9b2027c56bd163dbc883e037b16c3469614752e148d219c61/wrapt-2.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f32fe639c39561ccc187bcae17e9271be0eb45f1c2952510d2f29b33ab577347", size = 81182 }, + { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460 }, ] [[package]]