feat: add token-aware truncation to remote embedding path

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## [Unreleased]
 
+### Added
+
+- Remote embedding, reranking, and query expansion via OpenAI-compatible API
+  (vLLM, Ollama, OpenAI, etc.). Set `QMD_EMBED_API_URL` / `QMD_EMBED_API_MODEL`
+  (and optionally `QMD_RERANK_API_*` / `QMD_EXPAND_API_*`) env vars or add
+  the equivalent keys to `models:` in `index.yml`. Local generation and
+  tokenization are preserved via a hybrid routing layer. Includes circuit
+  breakers, dimension validation, and batch splitting.
+
 ### Fixed
 
 - Filesystem paths with special characters (`#`, `&`, spaces, `[]`, `()`, etc.)
@@ -108,7 +117,6 @@
 
 - Launcher: Rewrite `bin/qmd` as a Node-based shebang polyglot to fix global npm installation execution failures on Windows (#668 / #452), while supporting seamless fallback to Bun in Node-less environments.
 
-
 ## [2.5.1] - 2026-05-20
 
 ### Changes

README.md

@@ -942,6 +942,46 @@ Uses node-llama-cpp's `createRankingContext()` and `rankAndSort()` API for cross
 
 Used for generating query variations via `LlamaChatSession`.
 
+### Remote Embedding, Reranking & Query Expansion
+
+QMD can offload embedding, reranking, and query expansion to remote OpenAI-compatible servers (vLLM, Ollama, LM Studio, OpenAI, etc.) while keeping local generation and tokenization available for hybrid fallback.
+
+**Environment variables** (presence of `QMD_EMBED_API_URL` activates remote mode):
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `QMD_EMBED_API_URL` | Yes | Base URL, e.g. `http://gpu-host:8000/v1` |
+| `QMD_EMBED_API_MODEL` | Yes | Model name, e.g. `BAAI/bge-m3` |
+| `QMD_EMBED_API_KEY` | No | Bearer token for auth |
+| `QMD_RERANK_API_URL` | No | Rerank endpoint (defaults to embed URL) |
+| `QMD_RERANK_API_MODEL` | No | Rerank model name |
+| `QMD_RERANK_API_KEY` | No | Rerank auth (defaults to embed key) |
+| `QMD_EXPAND_API_URL` | No | Query expansion chat endpoint (defaults to embed URL) |
+| `QMD_EXPAND_API_MODEL` | No | Chat model for query expansion |
+| `QMD_EXPAND_API_KEY` | No | Query expansion auth (defaults to embed key) |
+
+**YAML config** (`~/.config/qmd/index.yml`):
+```yaml
+models:
+  embed_api_url: "http://gpu-host:8000/v1"
+  embed_api_model: "BAAI/bge-m3"
+  rerank_api_model: "BAAI/bge-reranker-v2-m3"
+  expand_api_url: "https://chat-host/v1"
+  expand_api_model: "qwen3-4b"
+```
+
+**Example with vLLM:**
+```sh
+# Start vLLM with an embedding model
+vllm serve BAAI/bge-m3 --task embed
+
+# Point QMD at it
+export QMD_EMBED_API_URL=http://localhost:8000/v1
+export QMD_EMBED_API_MODEL=BAAI/bge-m3
+qmd embed
+qmd query "your search query"
+```
+
 ## License
 
 MIT

bin/qmd

@@ -1,7 +1,13 @@
-#!/usr/bin/env node
+#!/home/nathan/.hermes/node/bin/node
 // 2>/dev/null; if command -v node >/dev/null 2>&1; then exec node "$0" "$@"; else exec bun "$0" "$@"; fi
 // Cross-platform launcher for qmd.
 //
+// NOTE (personal checkout, ubuntunix): the shebang is pinned to hermes Node 22
+// (ABI 127) because this checkout's native deps (better-sqlite3, sqlite-vec)
+// are kept built for that runtime, and the production consumers (remnic / RMO,
+// qmd-mcp) resolve `qmd` through this file via PATH. Portable installs (Nix
+// derivation / npm global) keep `#!/usr/bin/env node` -- see #381.
+//
 // Previously this was a POSIX shell script with `#!/bin/sh`, which meant npm
 // on Windows generated shims that tried to route through `/bin/sh` — a path
 // that doesn't exist on Windows, so `qmd` failed immediately after a global
@@ -142,6 +148,25 @@ const runner = useSourceMode ? sourceRunner : (runnerName === "node" ? "node" :
 const args = useSourceMode ? sourceArgs : [jsEntry, ...process.argv.slice(2)];
 const needsShell = (runner === "bun") && process.platform === "win32";
 
+// ABI tripwire (2026-09-12, incident: RMO warm-up looped 1800+ times on
+// ERR_DLOPEN_FAILED). When the resolved runner is Node, the Node the launcher
+// itself runs under is the Node (same PATH, same spawn) that will load the
+// native deps. If it is not the ABI-127 (Node 22) build target this checkout's
+// deps use, fail here with a message that says what happened, instead of dying
+// inside process.dlopen with a stack trace. Bun is exempt: Nix-managed bun
+// loads the current addon set without issue and has no Node-ABI constraint.
+if (runner === "node" && process.versions.modules !== "127") {
+  console.error(
+    `qmd: native deps in this checkout are built for Node 22 (ABI 127), but the ` +
+      `resolved Node runner reports ABI ${process.versions.modules} (Node ${process.version}).\n` +
+      `  If node_modules was just (re)installed by a different Node, rebuild the native ` +
+      `modules for Node 22:\n` +
+      `    cd ${pkgDir} && PATH="$HOME/.hermes/node/bin:$PATH" npm rebuild better-sqlite3\n` +
+      `  (or re-run the install with hermes Node 22 first on PATH).`
+  );
+  process.exit(1);
+}
+
 const child = spawn(runner, args, {
   stdio: "inherit",
   shell: needsShell,

src/cli/qmd.ts

@@ -33,6 +33,7 @@ import {
   formatDocForEmbedding,
   getEmbeddingFingerprint,
   chunkDocumentByTokens,
+  chunkDocumentByApproxTokens,
   clearCache,
   getCacheKey,
   getCachedResult,
@@ -81,7 +82,8 @@ import {
   type ReindexResult,
   type ChunkStrategy,
 } from "../store.js";
-import { disposeDefaultLlamaCpp, getDefaultLlamaCpp, setDefaultLlamaCpp, LlamaCpp, withLLMSession, pullModels, DEFAULT_MODEL_CACHE_DIR, resolveEmbedModel, resolveGenerateModel, resolveRerankModel, resolveModels, inspectGgufFile, isDarwinMetalMitigationActive } from "../llm.js";
+import { disposeDefaultLlamaCpp, getDefaultLLM, setDefaultLLM, getDefaultLlamaCpp, setDefaultLlamaCpp, LlamaCpp, withLLMSession, pullModels, DEFAULT_MODEL_CACHE_DIR, resolveEmbedModel, resolveGenerateModel, resolveRerankModel, resolveModels, inspectGgufFile, isDarwinMetalMitigationActive } from "../llm.js";
+import { createConfiguredLLM } from "../configured-llm.js";
 import {
   formatSearchResults,
   formatDocuments,
@@ -130,18 +132,19 @@ function getStore(): ReturnType<typeof createStore> {
   if (!store) {
     store = createStore(storeDbPathOverride);
     // Sync YAML config into SQLite store_collections so store.ts reads from DB
+    const activeModels = ensureModelsConfiguredForCli();
+    let config: CollectionConfig | undefined;
     try {
-      const activeModels = ensureModelsConfiguredForCli();
-      const config = loadConfig();
+      config = loadConfig();
       syncConfigToDb(store.db, config);
-      setDefaultLlamaCpp(new LlamaCpp({
-        embedModel: activeModels.embed,
-        generateModel: activeModels.generate,
-        rerankModel: activeModels.rerank,
-      }));
     } catch {
       // Config may not exist yet — that's fine, DB works without it
     }
+    setDefaultLLM(createConfiguredLLM(config?.models, {
+      embedModel: activeModels.embed,
+      generateModel: activeModels.generate,
+      rerankModel: activeModels.rerank,
+    }));
   }
   return store;
 }
@@ -311,7 +314,11 @@ function formatETA(seconds: number): string {
 
 
 // Check index health and print warnings/tips
-function checkIndexHealth(db: Database, model: string = resolveEmbedModelForCli()): void {
+function getActiveEmbedModelForCli(): string {
+  return getDefaultLLM().embedModelName;
+}
+
+function checkIndexHealth(db: Database, model: string = getActiveEmbedModelForCli()): void {
   const { needsEmbedding, totalDocs, daysStale } = getIndexHealth(db, model);
 
   // Warn if many docs need embedding
@@ -484,7 +491,7 @@ async function showStatus(): Promise<void> {
   // Overall stats
   const totalDocs = db.prepare(`SELECT COUNT(*) as count FROM documents WHERE active = 1`).get() as { count: number };
   const vectorCount = db.prepare(`SELECT COUNT(*) as count FROM content_vectors`).get() as { count: number };
-  const statusEmbedModel = resolveEmbedModelForCli();
+  const statusEmbedModel = getActiveEmbedModelForCli();
   const needsEmbedding = getHashesNeedingEmbedding(db, undefined, statusEmbedModel);
 
   // Most recent update across all collections
@@ -741,7 +748,7 @@ async function updateCollections(): Promise<void> {
   }
 
   // Check if any documents need embedding (show once at end)
-  const needsEmbedding = getHashesNeedingEmbedding(db);
+  const needsEmbedding = getHashesNeedingEmbedding(db, undefined, getActiveEmbedModelForCli());
   closeDb();
 
   console.log(`${c.green}✓ All collections updated.${c.reset}`);
@@ -1900,7 +1907,7 @@ async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, coll
   const orphanedContent = cleanupOrphanedContent(db);
 
   // Check if vector index needs updating
-  const needsEmbedding = getHashesNeedingEmbedding(db);
+  const needsEmbedding = getHashesNeedingEmbedding(db, undefined, getActiveEmbedModelForCli());
 
   progress.clear();
   console.log(`\nIndexed: ${indexed} new, ${updated} updated, ${unchanged} unchanged, ${removed} removed`);
@@ -1984,6 +1991,39 @@ async function vectorIndex(
   const storeInstance = getStore();
   const db = storeInstance.db;
 
+  // PR #629 follow-up: when HybridLLM/RemoteLLM is configured, the actual embedding
+  // model name comes from the LLM (e.g. "nomic-embed-text") rather than the local
+  // GGUF URI we got from CLI defaults. Override here so generateEmbeddings, fingerprint
+  // computation, and the pending-doc lookup all use the remote model identifier.
+  const llm = getDefaultLLM();
+  model = llm.embedModelName;
+
+  // Pre-flight probe: when remote embedding is configured, verify the remote
+  // endpoint actually works BEFORE writing any vectors. This prevents the
+  // silent-fallback failure mode where a misconfigured/unreachable remote
+  // backend would otherwise let qmd appear to "succeed" while writing
+  // local-model-tagged vectors (or no vectors at all). Belt-and-suspenders
+  // on top of RemoteLLM's call-time errors — fail fast at startup with a
+  // clear message rather than mid-batch.
+  if (llm.usesRemoteEmbedding) {
+    try {
+      const probe = await llm.embed("preflight probe", { model });
+      if (!probe || !Array.isArray(probe.embedding) || probe.embedding.length === 0) {
+        throw new Error(
+          `Pre-flight probe to remote embedder returned no embedding ` +
+          `(model=${model}). Aborting embed run to avoid silent fallback. ` +
+          `Check llama-server / remote endpoint health.`
+        );
+      }
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      throw new Error(
+        `Pre-flight probe to remote embedder FAILED (model=${model}): ${msg}. ` +
+        `Aborting embed run to avoid silent fallback.`
+      );
+    }
+  }
+
   if (force) {
     console.log(`${c.yellow}Force re-indexing: clearing all vectors...${c.reset}`);
   }
@@ -3753,11 +3793,15 @@ async function checkEmbeddingVectorSamples(db: Database, model: string, fingerpr
 
   const threshold = 0.0001;
   const mismatches: string[] = [];
+  const llm = getDefaultLLM();
+  const usesRemoteEmbedding = llm.usesRemoteEmbedding === true;
 
   await withLLMSession(async (session) => {
     for (const sample of samples) {
       const hashSeq = `${sample.hash}_${sample.seq}`;
-      const chunks = await chunkDocumentByTokens(sample.body, undefined, undefined, undefined, sample.path, undefined, session.signal);
+      const chunks = usesRemoteEmbedding
+        ? await chunkDocumentByApproxTokens(sample.body, undefined, undefined, undefined, sample.path, undefined, session.signal)
+        : await chunkDocumentByTokens(sample.body, undefined, undefined, undefined, sample.path, undefined, session.signal);
       const chunk = chunks[sample.seq];
       if (!chunk) {
         mismatches.push(`${shortHashSeq(hashSeq)}: chunk no longer exists`);
@@ -3868,7 +3912,15 @@ async function runDoctorDeviceChecks(nextSteps: string[]): Promise<void> {
   }
 
   try {
-    const device = await getDefaultLlamaCpp().getDeviceInfo({ allowBuild: false });
+    const llm = getDefaultLLM();
+    if (!(llm instanceof LlamaCpp)) {
+      if (process.stdout.isTTY) {
+        process.stdout.write(`\r${" ".repeat(crashHint.length)}\r`);
+      }
+      console.log(`  ${c.dim}device probe unavailable for non-local LLM backend${c.reset}`);
+      return;
+    }
+    const device = await llm.getDeviceInfo({ allowBuild: false });
     if (process.stdout.isTTY) {
       process.stdout.write(`\r${" ".repeat(crashHint.length)}\r`);
     }
@@ -3934,7 +3986,7 @@ async function showDoctor(): Promise<void> {
   const db = storeInstance.db;
   const pkg = readPackageJson();
   const activeModels = resolveModelsForCli();
-  const embedModel = activeModels.embed;
+  const embedModel = getActiveEmbedModelForCli();
   const fingerprint = getEmbeddingFingerprint(embedModel);
   const nextSteps: string[] = [];
 

src/collections.ts

@@ -40,6 +40,24 @@ export interface ModelsConfig {
   embed?: string;
   rerank?: string;
   generate?: string;
+  /** Remote embedding API base URL (e.g. http://gpu-host:8000/v1) */
+  embed_api_url?: string;
+  /** Remote embedding model name (e.g. BAAI/bge-m3) */
+  embed_api_model?: string;
+  /** Bearer token for remote embedding API */
+  embed_api_key?: string;
+  /** Remote rerank API base URL (defaults to embed_api_url) */
+  rerank_api_url?: string;
+  /** Remote rerank model name */
+  rerank_api_model?: string;
+  /** Bearer token for remote rerank API */
+  rerank_api_key?: string;
+  /** Remote query expansion API base URL */
+  expand_api_url?: string;
+  /** Remote query expansion chat model name */
+  expand_api_model?: string;
+  /** Bearer token for remote query expansion API */
+  expand_api_key?: string;
 }
 
 /**

src/configured-llm.ts

@@ -0,0 +1,21 @@
+import type { ModelsConfig } from "./collections.js";
+import { HybridLLM } from "./hybrid-llm.js";
+import { LlamaCpp, type LLM, type LlamaCppConfig } from "./llm.js";
+import { RemoteLLM, remoteConfigFromEnv } from "./remote-llm.js";
+
+/**
+ * Build the LLM backend implied by config/env.
+ *
+ * Remote embedding is opt-in via remoteConfigFromEnv(). When configured, a
+ * HybridLLM keeps local generation/tokenization/fallback behavior while routing
+ * remote-capable operations through the OpenAI-compatible API.
+ */
+export function createConfiguredLLM(
+  models?: ModelsConfig,
+  localConfig: LlamaCppConfig = {},
+): LLM {
+  const remoteConfig = remoteConfigFromEnv(models);
+  const local = new LlamaCpp(localConfig);
+  if (!remoteConfig) return local;
+  return new HybridLLM(new RemoteLLM(remoteConfig), local);
+}