Add remote embedding, reranking, and query expansion support

CHANGELOG.md

@@ -6,6 +6,13 @@
 
 - Added Oxlint lint fence.
 
+- Remote embedding, reranking, and query expansion via OpenAI-compatible API
+  (vLLM, Ollama, OpenAI, etc.). Set `QMD_EMBED_API_URL` / `QMD_EMBED_API_MODEL`
+  (and optionally `QMD_RERANK_API_*` / `QMD_EXPAND_API_*`) env vars or add
+  the equivalent keys to `models:` in `index.yml`. Local generation and
+  tokenization are preserved via a hybrid routing layer. Includes circuit
+  breakers, dimension validation, and batch splitting.
+
 ## [2.8.3] - 2026-08-16
 
 ### Security

README.md

@@ -1296,6 +1296,41 @@ Uses node-llama-cpp's `createRankingContext()` and `rankAndSort()` API for cross
 
 Used for generating query variations via `LlamaChatSession`.
 
+### Remote Embedding & Reranking
+
+QMD can offload embedding and reranking to a remote OpenAI-compatible server (vLLM, Ollama, LM Studio, OpenAI, etc.) while keeping query expansion local.
+
+**Environment variables** (presence of `QMD_EMBED_API_URL` activates remote mode):
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `QMD_EMBED_API_URL` | Yes | Base URL, e.g. `http://gpu-host:8000/v1` |
+| `QMD_EMBED_API_MODEL` | Yes | Model name, e.g. `BAAI/bge-m3` |
+| `QMD_EMBED_API_KEY` | No | Bearer token for auth |
+| `QMD_RERANK_API_URL` | No | Rerank endpoint (defaults to embed URL) |
+| `QMD_RERANK_API_MODEL` | No | Rerank model name |
+| `QMD_RERANK_API_KEY` | No | Rerank auth (defaults to embed key) |
+
+**YAML config** (`~/.config/qmd/index.yml`):
+```yaml
+models:
+  embed_api_url: "http://gpu-host:8000/v1"
+  embed_api_model: "BAAI/bge-m3"
+  rerank_api_model: "BAAI/bge-reranker-v2-m3"
+```
+
+**Example with vLLM:**
+```sh
+# Start vLLM with an embedding model
+vllm serve BAAI/bge-m3 --task embed
+
+# Point QMD at it
+export QMD_EMBED_API_URL=http://localhost:8000/v1
+export QMD_EMBED_API_MODEL=BAAI/bge-m3
+qmd embed
+qmd query "your search query"
+```
+
 ## License
 
 MIT

src/cli/qmd.ts

@@ -86,7 +86,9 @@ import {
   type ReindexResult,
   type ChunkStrategy,
 } from "../store.js";
-import { disposeDefaultLlamaCpp, getDefaultLlamaCpp, setDefaultLlamaCpp, LlamaCpp, withLLMSession, pullModels, DEFAULT_MODEL_CACHE_DIR, resolveEmbedModel, resolveGenerateModel, resolveRerankModel, resolveModels, inspectGgufFile, isDarwinMetalMitigationActive } from "../llm.js";
+import { disposeDefaultLlamaCpp, getDefaultLLM, setDefaultLLM, LlamaCpp, withLLMSession, pullModels, DEFAULT_EMBED_MODEL_URI, DEFAULT_GENERATE_MODEL_URI, DEFAULT_RERANK_MODEL_URI, DEFAULT_MODEL_CACHE_DIR, resolveEmbedModel, resolveGenerateModel, resolveRerankModel, resolveModels, inspectGgufFile, isDarwinMetalMitigationActive } from "../llm.js";
+import { RemoteLLM, remoteConfigFromEnv } from "../remote-llm.js";
+import { HybridLLM } from "../hybrid-llm.js";
 import {
   formatSearchResults,
   formatDocuments,
@@ -159,12 +161,21 @@ function getStore(): ReturnType<typeof createStore> {
       // Untrusted project-local custom model URIs must not be loaded; status
       // still displays the YAML values via resolveModelsForCli (#889).
       const modelsForLlm = localConfigIsFullyTrusted() ? activeModels : resolveModels();
-      const llm = new LlamaCpp({
+      const localLlm = new LlamaCpp({
         embedModel: modelsForLlm.embed,
         generateModel: modelsForLlm.generate,
         rerankModel: modelsForLlm.rerank,
       });
-      setDefaultLlamaCpp(llm);
+
+      // Environment variables always take precedence. Project-local remote
+      // endpoints are loaded only after the same trust gate as custom models.
+      const remoteConfig = localConfigIsFullyTrusted()
+        ? remoteConfigFromEnv(config.models)
+        : remoteConfigFromEnv();
+      const llm = remoteConfig
+        ? new HybridLLM(new RemoteLLM(remoteConfig), localLlm)
+        : localLlm;
+      setDefaultLLM(llm);
       store.llm = llm;
     } catch {
       // Config may not exist yet — that's fine, DB works without it
@@ -667,7 +678,6 @@ async function showStatus(): Promise<void> {
     console.log(`  Generation:  ${hfLink(activeModels.generate)}`);
   }
 
-
   // Tips section
   const tips: string[] = [];
 
@@ -2165,6 +2175,10 @@ async function vectorIndex(
   const storeInstance = getStore();
   const db = storeInstance.db;
 
+  // Use the actual configured backend model (which may be remote) for the
+  // embedding fingerprint and pending-work check.
+  model = storeInstance.llm?.embedModelName ?? getDefaultLLM().embedModelName;
+
   // Exclusive process lock — concurrent embeds race on vectors_vec (#825)
   const embedLock = tryAcquireEmbedLock(embedLockPathForDb(getDbPath()));
   if (!embedLock) {
@@ -4066,7 +4080,12 @@ async function runDoctorDeviceChecks(nextSteps: string[]): Promise<void> {
   }
 
   try {
-    const device = await getDefaultLlamaCpp().getDeviceInfo({ allowBuild: false });
+    const defaultLlm = getDefaultLLM();
+    if (!(defaultLlm instanceof LlamaCpp)) {
+      doctorCheck("device probe", false, "unavailable for the configured remote/hybrid LLM backend");
+      return;
+    }
+    const device = await defaultLlm.getDeviceInfo({ allowBuild: false });
     if (process.stdout.isTTY) {
       process.stdout.write(`\r${" ".repeat(crashHint.length)}\r`);
     }

src/collections.ts

@@ -40,6 +40,18 @@ export interface ModelsConfig {
   embed?: string;
   rerank?: string;
   generate?: string;
+  /** Remote embedding API base URL (e.g. http://gpu-host:8000/v1) */
+  embed_api_url?: string;
+  /** Remote embedding model name (e.g. BAAI/bge-m3) */
+  embed_api_model?: string;
+  /** Bearer token for remote embedding API */
+  embed_api_key?: string;
+  /** Remote rerank API base URL (defaults to embed_api_url) */
+  rerank_api_url?: string;
+  /** Remote rerank model name */
+  rerank_api_model?: string;
+  /** Bearer token for remote rerank API */
+  rerank_api_key?: string;
 }
 
 /**

src/hybrid-llm.ts

@@ -0,0 +1,83 @@
+/**
+ * hybrid-llm.ts - Compositor that routes LLM operations between remote and local backends
+ *
+ * Embed/rerank → remote (GPU-heavy, benefits from offloading)
+ * Generate → local LlamaCpp
+ * ExpandQuery → remote when expandApiModel is configured, otherwise local LlamaCpp
+ * tokenize/countTokens → local LlamaCpp (CPU-cheap, needed for chunking)
+ */
+
+import type {
+  LLM,
+  EmbedOptions,
+  EmbeddingResult,
+  GenerateOptions,
+  GenerateResult,
+  ModelInfo,
+  Queryable,
+  RerankDocument,
+  RerankOptions,
+  RerankResult,
+} from "./llm.js";
+import type { Token as LlamaToken } from "node-llama-cpp";
+import { RemoteLLM } from "./remote-llm.js";
+
+export class HybridLLM implements LLM {
+  constructor(
+    private readonly remote: LLM,
+    private readonly local: LLM,
+  ) {}
+
+  get embedModelName(): string {
+    return this.remote.embedModelName;
+  }
+
+  get usesRemoteEmbedding(): boolean {
+    return this.remote.usesRemoteEmbedding === true;
+  }
+
+  // Route to remote
+  embed(text: string, options?: EmbedOptions): Promise<EmbeddingResult | null> {
+    return this.remote.embed(text, options);
+  }
+
+  embedBatch(texts: string[], options?: EmbedOptions): Promise<(EmbeddingResult | null)[]> {
+    return this.remote.embedBatch(texts, options);
+  }
+
+  rerank(query: string, documents: RerankDocument[], options?: RerankOptions): Promise<RerankResult> {
+    return this.remote.rerank(query, documents, options);
+  }
+
+  // Route to local
+  generate(prompt: string, options?: GenerateOptions): Promise<GenerateResult | null> {
+    return this.local.generate(prompt, options);
+  }
+
+  tokenize(text: string): Promise<readonly LlamaToken[]> {
+    return this.local.tokenize(text);
+  }
+
+  detokenize(tokens: readonly LlamaToken[]): Promise<string> {
+    return this.local.detokenize(tokens);
+  }
+
+  /**
+   * Route expandQuery to remote when the remote backend supports it
+   * (i.e., RemoteLLM with expandApiModel configured), otherwise fall back to local.
+   */
+  expandQuery(query: string, options?: { context?: string; includeLexical?: boolean; intent?: string }): Promise<Queryable[]> {
+    if (this.remote instanceof RemoteLLM && this.remote.supportsExpand) {
+      return this.remote.expandQuery(query, options);
+    }
+    return this.local.expandQuery(query, options);
+  }
+
+  modelExists(model: string): Promise<ModelInfo> {
+    return this.local.modelExists(model);
+  }
+
+  async dispose(): Promise<void> {
+    await Promise.all([this.remote.dispose(), this.local.dispose()]);
+  }
+}