Migrate document extraction to xberg 1.0

.github/workflows/release.yml

@@ -133,10 +133,36 @@ jobs:
             compat: true
             soft: true
             march: x86-64-v2
+          # macos_floor is the OS this asset actually runs on, and every bundled
+          # dylib is checked against it. `uv sync` resolves wheels for the runner,
+          # so numpy/scipy arrive as macosx_14_0 and 14.0 is the honest floor here.
           - os: macos-latest
             asset_name: lilbee-macos-arm64
             backend: metal
-          # Intel Mac dropped: macos-13 hosted runners stay queued; pip wheel covers them.
+            macos_floor: "14.0"
+          # Intel Mac: built natively on macos-15-intel, GitHub's Intel image
+          # (Nuitka can't cross-compile arch from the arm64 runner). Soft because
+          # the Intel image can queue longer than the arm64 pool -- this asset must
+          # never block the release fan-out. The install below pins wheel resolution
+          # to the x86_64 target rather than the runner, so the deps come in at their
+          # oldest macOS build (pyarrow's 12.0 is the binding one) and the AVX
+          # baseline reaches every Ivy Bridge / Haswell Mac, which cap at macOS 12.
+          - os: macos-15-intel
+            asset_name: lilbee-macos-x86_64
+            backend: cpu
+            soft: true
+            macos_floor: "12.0"
+          # Pre-Haswell-safe Intel Mac build: same self-contained model as the
+          # linux/windows compat cells (stock lancedb swapped for +compat, launcher
+          # held to x86-64-v2). Covers the AVX-but-no-AVX2 tail (e.g. 2013 Mac Pro)
+          # and guards against an AVX2-baselined stock lancedb SIGILL.
+          - os: macos-15-intel
+            asset_name: lilbee-compat-macos-x86_64
+            backend: cpu
+            compat: true
+            soft: true
+            march: x86-64-v2
+            macos_floor: "12.0"
           - os: windows-latest
             asset_name: lilbee-windows-x86_64.exe
             backend: vulkan
@@ -218,13 +244,51 @@ jobs:
           save: ${{ github.ref_type != 'tag' }}
 
       - name: Install dependencies
+        if: matrix.os != 'macos-15-intel'
         run: uv sync --extra release && uv pip install 'nuitka[onefile]>=4.0,<5' pip
 
+      # Intel Mac: `uv sync` can't run here -- stock lancedb 0.33.0 ships no
+      # macosx_x86_64 wheel. Resolve it from the +compat index (PEP 440 drop-in),
+      # then install the rest of [release] fresh from PyPI. CMAKE_ARGS + deployment
+      # target hold any throwaway llama-cpp-python sdist build at the cpu/x86_64
+      # baseline; the dedicated step below builds the llama-cpp-python that actually
+      # ships. The cell stays soft so an install failure never blocks the fan-out.
+      #
+      # --python-platform resolves wheels for the x86_64 target instead of the
+      # runner. Without it uv picks the newest wheel the macOS 15 image accepts
+      # (numpy/scipy macosx_14_0) and bundles a binary that dies on macOS 12-13,
+      # even though both ship older-built wheels that run just as well here.
+      - name: Install dependencies (Intel Mac)
+        if: matrix.os == 'macos-15-intel'
+        shell: bash
+        env:
+          MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_floor }}
+        run: |
+          set -euxo pipefail
+          eval "$(BACKEND=cpu TARGET_ARCH=x86_64 tools/wheel-build/cmake_args.sh)"
+          export CMAKE_ARGS
+          uv venv
+          uv pip install --python-platform x86_64-apple-darwin \
+            --extra-index-url https://lilbee.sh/compat/ \
+            lancedb==0.33.0+compat
+          uv pip install --python-platform x86_64-apple-darwin \
+            '.[release]' 'nuitka[onefile]>=4.0,<5' pip
+
       - name: Verify extras installed in build venv
         shell: bash
         # --no-sync is load-bearing: re-syncing reinstalls python-multipart and shadows the shim.
         run: uv run --no-sync python -c "import crawl4ai, litellm, spacy, graspologic_native; print('extras OK')"
 
+      # Nuitka bundles these dylibs, and dyld enforces each one's minos separately
+      # from the launcher's, so one dep built on a newer runner silently ships a
+      # binary that dies at load. Fail here rather than on the user's Mac.
+      - name: Verify macOS floor
+        if: runner.os == 'macOS'
+        shell: bash
+        env:
+          MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_floor }}
+        run: bash tools/wheel-build/check_macos_floor.sh
+
       # compat cell only: swap the synced stock lancedb for the pre-Haswell build.
       # --no-deps leaves the rest of the venv intact, so nothing leaves this cell.
       # Pinned to the latest stable lancedb for Python (0.33), served by the fork as
@@ -248,13 +312,19 @@ jobs:
           # engine cache separate from bare-runner builds.
           cache-env: ${{ matrix.container || matrix.os }}
         env:
-          MACOSX_DEPLOYMENT_TARGET: "11.0"
+          MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_floor }}
 
       - name: Build executable
         shell: bash
         env:
           ASSET_NAME: ${{ matrix.asset_name }}
           PRODUCT_VERSION: ${{ env.LILBEE_NUITKA_VERSION }}
+          # Pin Nuitka's clang output to this asset's floor. Unset, clang defaults the
+          # deployment target to the build host (macOS 15), which would stamp the
+          # launcher minos=15 and refuse to launch below it. This only pins the
+          # launcher; the bundled dylibs carry their own minos, which is what the
+          # "Verify macOS floor" step above checks. No-op off macOS.
+          MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_floor }}
           # compat cell only: hold Nuitka's C output to the pre-Haswell baseline so
           # the launcher layer (not just lancedb) runs on old silicon. Empty elsewhere.
           CFLAGS: ${{ matrix.march && format('-march={0}', matrix.march) || '' }}

.gitignore

@@ -30,7 +30,7 @@ CLAUDE.md
 docs/plans/
 demo.cast
 docs/superpowers/
-.kreuzberg/
+.xberg/
 .claude/
 .dolt/
 .doltcfg/

AGENTS.md

@@ -140,7 +140,7 @@ CLI also accepts `--model` / `-m` for chat model, `--data-dir` / `-d`, `--ocr-ti
 - Type checking: `mypy` with strict settings
 - **No em dashes (—)** — use periods or commas instead
 - **No divider comments** (`# ------`) for grouping — use modules or classes instead
-- **Don't scatter functions between constants.** Within a module (or a topical block in one), keep the module-level constants contiguous and the functions/classes contiguous. A `def` wedged into the middle of a run of `FOO = ...` / `BAR = ...` lines breaks the eye's scan. A helper that formats or validates a small group of constants may follow that group directly (the constants, then their one helper), but don't interleave: `A = ...`, `def f(): ...`, `B = ...`, `def g(): ...`, `C = ...` is wrong; group it as `A = ...`, `B = ...`, `C = ...`, `def f(): ...`, `def g(): ...`. Order by concern (all the "skip" constants, then the skip helpers; all the "doc table" constants, then the doc-table dataclass), not by when each was added.
+- **Canonical module layout — group by kind, don't scatter.** A module reads top-to-bottom in this order: module docstring → `from __future__` → imports → module-level constants & data tables → enums → exception classes → data containers (`dataclass` / `TypedDict` / `NamedTuple` / `BaseModel`) → classes → functions. Within each band, order by concern, not by when each symbol was added. The reader should never have to scan past a `class` or `def` to find the next constant, nor past a constant to find the next class. A constant assignment (`FOO = ...`, `BAR: T = ...`) appearing AFTER the first `def`/`class` in a module is the scatter smell — hoist it into the constants band at the top. A `def` wedged into a run of `FOO = ...` / `BAR = ...` lines is the same smell from the other side. The one allowed exception: a single small helper that exists only to build or validate one adjacent group of constants may sit directly after that group (the constants, then their one helper) — but never interleaved (`A = ...`, `def f(): ...`, `B = ...` is always wrong). When in doubt, constants and data tables go at the top, contiguous; everything callable goes below, contiguous.
 - **Annotate `isinstance` guards** with a comment explaining why the check is needed (e.g. untyped frontmatter, untyped SDK response). NEVER use `isinstance(self.app, LilbeeApp)` style host-narrowing in production: if a screen / widget needs LilbeeApp-specific access, declare `app: LilbeeApp  # type: ignore[assignment]` at the class scope (mypy narrows; runtime is unchanged). Tests host via `tests/_lilbee_app_test_host.LilbeeAppHost` (a LilbeeApp subclass with `_test_skip_auto_init = True`), never `App[None]` / `class _PlainApp(App)`.
 - Use literal unicode chars (`★`) not escapes (`\u2605`); extract to named constants
 - **No filterwarnings** without explicit user approval; fix warnings at the source
@@ -190,7 +190,7 @@ Default: **every import lives at module top**, ordered stdlib, third-party, loca
 **Permitted reasons for a function-local import:**
 
 1. **Circular import.** Module A's top-level already imports module B, and module B needs a symbol from A. Put A's import of B inside the one function in B that needs it, with a comment `# circular: B -> A via <symbol>`.
-2. **Heavy third-party lib.** Module-top import takes **>50 ms measured by `python -X importtime`** or loads native libraries. Known-heavy libs in this project: `litellm` (provider SDK fanout), `lancedb` (arrow + datafusion), `kreuzberg` (OCR stack), `gguf` (numpy), `sentence_transformers`, `spacy`, `crawl4ai`, `textual` when imported outside the TUI screen modules. (The local inference engine is the out-of-process `llama-server`, reached over HTTP, so it adds no heavy Python import.) Use importtime before declaring a lib heavy.
+2. **Heavy third-party lib.** Module-top import takes **>50 ms measured by `python -X importtime`** or loads native libraries. Known-heavy libs in this project: `litellm` (provider SDK fanout), `lancedb` (arrow + datafusion), `xberg` (OCR stack), `gguf` (numpy), `sentence_transformers`, `spacy`, `crawl4ai`, `textual` when imported outside the TUI screen modules. (The local inference engine is the out-of-process `llama-server`, reached over HTTP, so it adds no heavy Python import.) Use importtime before declaring a lib heavy.
 3. **CLI startup path.** The import lives inside a Typer command body so `lilbee --help` stays fast. Treat this as a sub-case of (2): the CLI loader stays lean so unused subcommands don't pay for their dependencies.
 
 **Never lazy-import the following:**
@@ -390,6 +390,7 @@ work; the greps below remain the full set a reviewer still runs by eye.
 - `grep -rn "App\[None\]\|class _PlainApp\b\|class _BareApp\b" tests/` — test host that bypasses LilbeeApp. Migrate to LilbeeAppHost.
 - `grep -cE "isinstance\(.*widget, (Input|Checkbox|Select|TextArea)\)" src/` followed by "if more than one site" — type-keyed widget routing. Replace with a `dict[type, Callable]`.
 - `grep -rnE "^\s*global \w+" src/` — module-level mutable globals. Encapsulate on a class.
+- `for f in $(git diff --name-only main -- 'src/**/*.py'); do awk 'NR>1 && /^[A-Z_]+ *[:=]/ && seen {print FILENAME": "$0} /^(def |class )/{seen=1}' "$f"; done` — module-level constant assigned AFTER the first `def`/`class` (the scatter smell). Hoist it into the constants band at the top. See the canonical-module-layout rule in Code Style.
 - `grep -rnE "def \w+\(.*\) -> .*:\s*$" src/ | xargs -I{} awk 'def_lines>20'` (proxy: scan modules >700 LOC) — functions over ~20 lines need a split into named sub-helpers.
 - `grep -rn "from lilbee\.\(modelhub\|providers\|data\|retrieval\|wiki\|crawler\)" src/lilbee/catalog src/lilbee/core` — upward imports from foundation layers. catalog and core never import from anything above them.
 - `grep -rn "from lilbee\.\(retrieval\|wiki\|crawler\|cli\|server\)" src/lilbee/data src/lilbee/modelhub src/lilbee/providers` — Layer-2 modules importing Layer-3+. Invert via callback or move the helper down.