From 6b2970277927bd44a2f2df2a45cea4aa14bccac7 Mon Sep 17 00:00:00 2001
From: NotXf1le <89696340+NotXf1le@users.noreply.github.com>
Date: Wed, 23 Sep 2026 20:31:56 +0200
Subject: [PATCH 1/4] Batch llama.cpp tokenization requests

---
 TODO.md                  |   2 +-
 src/llama-cpp.ts         | 105 ++++++++++++++++++++---
 tests/llama-cpp.test.mjs | 177 ++++++++++++++++++++++++++++++++++++++-
 3 files changed, 271 insertions(+), 13 deletions(-)

diff --git a/TODO.md b/TODO.md
index fecc4a3..9ad3777 100644
--- a/TODO.md
+++ b/TODO.md
@@ -2,5 +2,5 @@
 
 ## Deferred
 
-- Tokenize the prompt and prompt-plus-candidate inputs with bounded parallelism in the `llama.cpp` adapter instead of awaiting every `/tokenize` request sequentially. Preserve input order and exact boundary-aware tokenization; make concurrency configurable and measure transport overhead separately from server processing. Consider a future native batch-tokenization endpoint returning one token array per input, because stock `/tokenize` does not provide that response shape.
+- Consider removing `addSpecialTokens` from the `llama.cpp` options in a future release.
 - Add native `llama.cpp` support for requesting raw log probabilities for multiple arbitrary token IDs in one request (`logprob_token_ids`). Prefer zero-token generation (`n_predict: 0`) and calculate every requested token's log probability from the full-vocabulary softmax. Keep the current top-N plus exact forced-token fallback path for servers without the extension; an explicitly selected bulk mode must fail clearly when unsupported instead of silently changing protocols.
diff --git a/src/llama-cpp.ts b/src/llama-cpp.ts
index fe62fa1..c93804e 100644
--- a/src/llama-cpp.ts
+++ b/src/llama-cpp.ts
@@ -42,6 +42,16 @@ interface Branch {
   readonly score: number;
 }
 
+class UnsupportedBatchTokenization extends Error {}
+class BatchTokenizationTooLarge extends Error {}
+
+const TOKENIZE_CONCURRENCY = 16;
+
+function isBatchFormatRejection(message: string): boolean {
+  return /(?:content|input).{0,40}(?:must|expected|requires?).{0,40}(?:string|array)|(?:expected|requires?).{0,40}string.{0,40}content|(?:unsupported|invalid).{0,40}(?:mixed|array)|(?:mixed|array).{0,40}(?:not supported)/i
+    .test(message);
+}
+
 function endpoints(baseURL: string, model?: string): Endpoints {
   requireText(baseURL, "baseURL");
   const url = new URL(baseURL);
@@ -77,7 +87,8 @@ function requestHeaders(value: unknown): Readonly<Record<string, string>> {
 }
 
 async function post(fetchImpl: typeof globalThis.fetch, url: string,
-  headers: Readonly<Record<string, string>>, body: unknown, signal?: AbortSignal): Promise<unknown> {
+  headers: Readonly<Record<string, string>>, body: unknown, signal?: AbortSignal,
+  batchTokenize = false): Promise<unknown> {
   signal?.throwIfAborted();
   let response: Response;
   try {
@@ -90,6 +101,12 @@ async function post(fetchImpl: typeof globalThis.fetch, url: string,
   }
   signal?.throwIfAborted();
   if (!response.ok) {
+    if (batchTokenize && response.status === 413) throw new BatchTokenizationTooLarge();
+    if (batchTokenize && [400, 415, 422].includes(response.status)) {
+      const message = await response.text();
+      signal?.throwIfAborted();
+      if (isBatchFormatRejection(message)) throw new UnsupportedBatchTokenization();
+    }
     throw new ScoringError(`llama.cpp returned HTTP ${response.status} for ${new URL(url).pathname}.`);
   }
   try {
@@ -131,6 +148,29 @@ function parseTokenization(value: unknown): number[] {
   return tokenIds(value.tokens);
 }
 
+function parseBatchedTokenization(value: unknown, expectedParts: number): number[][] {
+  if (!isRecord(value) || !Array.isArray(value.tokens)) {
+    throw new ScoringError("llama.cpp returned an invalid tokenization.");
+  }
+  const parts: number[][] = [[]];
+  let markers = 0;
+  for (const id of value.tokens) {
+    if (id === -1) {
+      markers++;
+      parts.push([]);
+    } else if (isCount(id)) {
+      parts[parts.length - 1]!.push(id);
+    } else {
+      throw new ScoringError("llama.cpp returned an invalid batched token ID.");
+    }
+  }
+  if (markers === 0 && parts[0]!.length > 0) throw new UnsupportedBatchTokenization();
+  if (markers !== expectedParts - 1 || parts.some((part) => part.length === 0)) {
+    throw new ScoringError("llama.cpp returned invalid batched tokenization boundaries.");
+  }
+  return parts;
+}
+
 function parseDetokenization(value: unknown): string {
   if (!isRecord(value) || typeof value.content !== "string") {
     throw new ScoringError("llama.cpp returned an invalid detokenization.");
@@ -236,6 +276,7 @@ export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
   if (typeof fetchImpl !== "function") throw new TypeError("A fetch implementation is required.");
   const urls = endpoints(baseURL, model);
   const headers = requestHeaders(options.headers);
+  let batchUnsupported = false;
   const score: Scorer = async ({ prompt, candidates, images, signal }) => {
     const hasImages = images !== undefined && images.length > 0;
     if (hasImages && mode !== "labels") {
@@ -248,13 +289,56 @@ export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
     const imageSupport = hasImages
       ? parseImageSupport(await get(fetchImpl, urls.props, headers, signal))
       : undefined;
-    const encoded: number[][] = [];
-    for (const content of [prompt, ...candidates.map((candidate) => prompt + candidate)]) {
-      const response = await post(fetchImpl, urls.tokenize, headers, {
-        content, add_special: addSpecialTokens, ...(model === undefined ? {} : { model }),
-      }, signal);
-      signal?.throwIfAborted();
-      encoded.push(parseTokenization(response));
+    const contents = [prompt, ...candidates.map((candidate) => prompt + candidate)];
+    let requests = hasImages ? 1 : 0;
+    const tokenizeIndividually = async (): Promise<number[][]> => {
+      const result: number[][] = new Array(contents.length);
+      let next = 0;
+      let failed = false;
+      await Promise.all(Array.from({ length: Math.min(TOKENIZE_CONCURRENCY, contents.length) },
+        async () => {
+          try {
+            while (!failed && next < contents.length) {
+              const index = next++;
+              signal?.throwIfAborted();
+              requests++;
+              const response = await post(fetchImpl, urls.tokenize, headers, {
+                content: contents[index], add_special: addSpecialTokens,
+                ...(model === undefined ? {} : { model }),
+              }, signal);
+              signal?.throwIfAborted();
+              result[index] = parseTokenization(response);
+            }
+          } catch (error) {
+            failed = true;
+            throw error;
+          }
+        }));
+      return result;
+    };
+    let encoded: number[][];
+    if (addSpecialTokens || batchUnsupported) {
+      encoded = await tokenizeIndividually();
+    } else {
+      try {
+        requests++;
+        const content: (string | number)[] = [];
+        for (const part of contents) {
+          if (content.length > 0) content.push(-1);
+          content.push(part);
+        }
+        const response = await post(fetchImpl, urls.tokenize, headers, {
+          content, add_special: false, ...(model === undefined ? {} : { model }),
+        }, signal, true);
+        signal?.throwIfAborted();
+        encoded = parseBatchedTokenization(response, contents.length);
+      } catch (error) {
+        if (!(error instanceof UnsupportedBatchTokenization || error instanceof BatchTokenizationTooLarge)) {
+          throw error;
+        }
+        if (error instanceof UnsupportedBatchTokenization) batchUnsupported = true;
+        encoded = await tokenizeIndividually();
+      }
     }
 
     const prefix = encoded[0]!;
@@ -264,7 +348,6 @@ export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
     let promptTokens = 0;
     let cachedTokens: number | null = 0;
     let completionTokens = 0;
-    let requests = encoded.length + (hasImages ? 1 : 0);
 
     let root = treeRoot;
     const rootSuffix: number[] = [];
@@ -330,7 +413,9 @@ export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
           prompt: promptValue,
           ...(model === undefined ? {} : { model }),
           n_predict: 1,
-          n_probs: collectSiblings ? 64 : 1,
+          n_probs: collectSiblings
+            ? Math.min(256, Math.max(64, children.length * 16))
+            : 1,
           post_sampling_probs: false,
           backend_sampling: false,
           samplers: ["top_k"],
diff --git a/tests/llama-cpp.test.mjs b/tests/llama-cpp.test.mjs
index 3b68c5c..b98199d 100644
--- a/tests/llama-cpp.test.mjs
+++ b/tests/llama-cpp.test.mjs
@@ -15,7 +15,7 @@ const decode = (tokens) => new TextDecoder().decode(
 );
 
 function fixture({ tokenizer = encode, detokenizer = decode, logprob = () => -1,
-  topTokenIds = (target) => [target], transform,
+  topTokenIds = (target) => [target], transform, batchTokenization,
   props = { modalities: { vision: true }, media_marker: "<__media_test__>" } } = {}) {
   const calls = [];
   const fetch = async (url, init) => {
@@ -24,7 +24,14 @@ function fixture({ tokenizer = encode, detokenizer = decode, logprob = () => -1,
     const body = init.body === undefined ? undefined : JSON.parse(init.body);
     calls.push({ url: parsedURL, path, body, headers: init.headers, signal: init.signal });
     if (path === "/props") return json(props);
-    if (path === "/tokenize") return json({ tokens: tokenizer(body.content, body.add_special) });
+    if (path === "/tokenize") {
+      if (Array.isArray(body.content)) {
+        if (batchTokenization) return batchTokenization(body);
+        return json({ tokens: body.content.flatMap((part) => part === -1
+          ? [-1] : tokenizer(part, body.add_special)) });
+      }
+      return json({ tokens: tokenizer(body.content, body.add_special) });
+    }
     if (path === "/detokenize") return json({ content: detokenizer(body.tokens) });
     if (path !== "/completion") return json({}, 404);
 
@@ -53,6 +60,7 @@ function json(value, status = 200) {
 }
 
 const completions = (f) => f.calls.filter((call) => call.path === "/completion");
+const tokenizations = (f) => f.calls.filter((call) => call.path === "/tokenize");
 
 test("maps label scores back to choice keys and reuses sibling logprobs", async () => {
   const a = encode("A")[0];
@@ -73,6 +81,168 @@ test("maps label scores back to choice keys and reuses sibling logprobs", async
   assert.equal(decision.usage.requests, f.calls.length);
   assert.equal(completions(f).length, 1);
   assert.equal(completions(f)[0].body.model, "local-model");
+  const [tokenization] = tokenizations(f);
+  assert.equal(tokenizations(f).length, 1);
+  const prompt = tokenization.body.content[0];
+  assert.deepEqual(tokenization.body.content, [prompt, -1, `${prompt}A`, -1, `${prompt}B`]);
+  assert.equal(tokenization.body.add_special, false);
+  assert.equal(tokenization.body.model, "local-model");
+  assert.equal(decision.usage.requests, 2);
+});
+
+test("scores eight labels with one completion when the wider top list covers them", async () => {
+  const labels = [..."ABCDEFGH"];
+  const ids = labels.map((label) => encode(label)[0]);
+  const scores = new Map(ids.map((id, index) => [id, index === 7 ? -0.1 : -index - 1]));
+  const f = fixture({
+    logprob: (id) => scores.get(id) ?? -10,
+    topTokenIds: (target, body) => body.n_probs >= 128 ? ids : [target],
+  });
+  const choose = fromLlamaCpp({ baseURL: "http://localhost:8080", fetch: f.fetch });
+  const choices = Object.fromEntries(labels.map((label) => [`option_${label}`, `Option ${label}`]));
+
+  const decision = await choose({ ...choiceRequest, choices });
+
+  assert.equal(decision.choice, "option_H");
+  assert.equal(decision.scores.option_H, -0.1);
+  assert.equal(completions(f).length, 1);
+  assert.equal(completions(f)[0].body.n_probs, 128);
+});
+
+test("falls back to bounded, out-of-order individual tokenizations and remembers incompatibility", async () => {
+  const labels = [..."ABCDEFGHIJKLMNOPQRST"];
+  const ids = labels.map((label) => encode(label)[0]);
+  const scores = new Map(ids.map((id, index) => [id, index === 7 ? -0.1 : -index - 1]));
+  const f = fixture({
+    batchTokenization: () => json({ tokens: [123] }),
+    logprob: (id) => scores.get(id) ?? -10,
+    topTokenIds: () => ids,
+  });
+  let active = 0;
+  let maximumActive = 0;
+  const completed = [];
+  const fetch = async (url, init) => {
+    const body = JSON.parse(init.body);
+    if (new URL(url).pathname === "/tokenize" && typeof body.content === "string") {
+      active++;
+      maximumActive = Math.max(maximumActive, active);
+      try {
+        await new Promise((resolve) => setTimeout(resolve, body.content.endsWith("A") ? 20 : 1));
+        const result = await f.fetch(url, init);
+        completed.push(body.content);
+        return result;
+      } finally {
+        active--;
+      }
+    }
+    return f.fetch(url, init);
+  };
+  const choose = fromLlamaCpp({ baseURL: "http://localhost:8080", fetch });
+  const choices = Object.fromEntries(labels.map((label) => [`option_${label}`, `Option ${label}`]));
+
+  const first = await choose({ ...choiceRequest, choices });
+  const second = await choose({ ...choiceRequest, choices });
+
+  for (const decision of [first, second]) {
+    assert.equal(decision.choice, "option_H");
+    assert.equal(decision.scores.option_H, -0.1);
+    assert.equal(decision.scores.option_A, -1);
+  }
+  assert.equal(first.usage.requests, 23);
+  assert.equal(second.usage.requests, 22);
+  assert.equal(tokenizations(f).filter((call) => Array.isArray(call.body.content)).length, 1);
+  assert.equal(tokenizations(f).filter((call) => typeof call.body.content === "string").length, 42);
+  assert.equal(maximumActive, 16);
+  assert.ok(completed.findIndex((content) => content.endsWith("B"))
+    < completed.findIndex((content) => content.endsWith("A")),
+  "a faster later request should complete before the first candidate");
+});
+
+test("retries a batch after a request-specific 413, with individual fallback for that invocation", async () => {
+  const f = fixture({
+    batchTokenization: () => json({ error: "payload too large" }, 413),
+    topTokenIds: () => [encode("A")[0], encode("B")[0]],
+  });
+  const choose = fromLlamaCpp({ baseURL: "http://localhost:8080", fetch: f.fetch });
+
+  const first = await choose(choiceRequest);
+  const second = await choose(choiceRequest);
+
+  assert.equal(first.choice, "wait");
+  assert.equal(second.choice, "wait");
+  assert.equal(first.usage.requests, 5);
+  assert.equal(second.usage.requests, 5);
+  assert.equal(tokenizations(f).filter((call) => Array.isArray(call.body.content)).length, 2);
+});
+
+test("remembers a format-specific HTTP 400 rejection for this chooser", async () => {
+  const f = fixture({ batchTokenization: () => json({ error: "content must be a string" }, 400) });
+  const choose = fromLlamaCpp({ baseURL: "http://localhost:8080", fetch: f.fetch });
+
+  assert.equal((await choose(choiceRequest)).choice, "wait");
+  assert.equal((await choose(choiceRequest)).choice, "wait");
+  assert.equal(tokenizations(f).filter((call) => Array.isArray(call.body.content)).length, 1);
+  assert.equal(tokenizations(f).filter((call) => typeof call.body.content === "string").length, 6);
+});
+
+test("addSpecialTokens retains independent tokenization with special insertion", async () => {
+  const f = fixture({ topTokenIds: () => [encode("A")[0], encode("B")[0]] });
+  const decision = await fromLlamaCpp({
+    baseURL: "http://localhost:8080", addSpecialTokens: true, fetch: f.fetch,
+  })(choiceRequest);
+
+  assert.equal(decision.choice, "wait");
+  assert.equal(decision.usage.requests, 4);
+  assert.equal(tokenizations(f).length, 3);
+  assert.ok(tokenizations(f).every((call) => typeof call.body.content === "string"
+    && call.body.add_special === true));
+});
+
+test("does not fall back on server, authorization, or rate-limit errors", async () => {
+  for (const status of [500, 401, 429]) {
+    const f = fixture({ batchTokenization: () => json({ error: "unavailable" }, status) });
+    const choose = fromLlamaCpp({ baseURL: "http://localhost:8080", fetch: f.fetch });
+    await assert.rejects(choose(choiceRequest), new RegExp(`HTTP ${status}`));
+    assert.equal(tokenizations(f).length, 1);
+    assert.equal(completions(f).length, 0);
+  }
+});
+
+test("does not fall back on network failure, invalid JSON, or abort", async () => {
+  const failure = new Error("network unavailable");
+  for (const batchTokenization of [
+    () => Promise.reject(failure),
+    () => new Response("not JSON", { headers: { "content-type": "application/json" } }),
+  ]) {
+    const f = fixture({ batchTokenization });
+    await assert.rejects(fromLlamaCpp({ baseURL: "http://localhost:8080", fetch: f.fetch })(choiceRequest));
+    assert.equal(tokenizations(f).length, 1);
+    assert.equal(completions(f).length, 0);
+  }
+  const controller = new AbortController();
+  const f = fixture();
+  const fetch = (url, init) => {
+    if (new URL(url).pathname === "/tokenize") controller.abort();
+    return f.fetch(url, init);
+  };
+  const choose = fromLlamaCpp({ baseURL: "http://localhost:8080", fetch });
+  await assert.rejects(choose({ ...choiceRequest, signal: controller.signal }),
+    (error) => error?.name === "AbortError");
+  assert.equal(tokenizations(f).length, 1);
+  assert.equal(completions(f).length, 0);
+});
+
+test("does not treat malformed batch token IDs or generic HTTP 400 as incompatibility", async () => {
+  for (const batchTokenization of [
+    () => json({ tokens: [1, -1, 2, -1, 3, -1] }),
+    () => json({ tokens: [1, -1, "bad", -1, 3] }),
+    () => json({ error: "invalid model" }, 400),
+  ]) {
+    const f = fixture({ batchTokenization });
+    await assert.rejects(fromLlamaCpp({ baseURL: "http://localhost:8080", fetch: f.fetch })(choiceRequest));
+    assert.equal(tokenizations(f).length, 1);
+    assert.equal(completions(f).length, 0);
+  }
 });
 
 test("scores image labels through native multimodal completion", async () => {
@@ -104,6 +274,7 @@ test("scores image labels through native multimodal completion", async () => {
   assert.deepEqual(decision.scores, { wait: -0.2, deploy: -1.5 });
   assert.equal(decision.usage.promptTokens, 246);
   assert.equal(decision.usage.requests, f.calls.length);
+  assert.equal(tokenizations(f).filter((call) => Array.isArray(call.body.content)).length, 1);
   const propsCall = f.calls.find((call) => call.path === "/props");
   assert.equal(propsCall.url.searchParams.get("model"), "vision/model");
   assert.equal(propsCall.signal, controller.signal);
@@ -375,6 +546,8 @@ test("reports a tokenization-boundary rollback", async () => {
   assert.equal(decision.choice, "first");
   assert.equal(decision.boundaryTokens, 1);
   assert.equal(completions(f)[0].body.prompt.length, 1);
+  assert.deepEqual(tokenizations(f).map((call) => call.body.content),
+    [["abc", -1, "abcA", -1, "abcB"]]);
 });
 
 test("rejects a probability attached to a different token", async () => {

From 2b7c5afa5eb2ce912a48638ffa6b8454cb144419 Mon Sep 17 00:00:00 2001
From: NotXf1le <89696340+NotXf1le@users.noreply.github.com>
Date: Wed, 23 Sep 2026 22:58:36 +0200
Subject: [PATCH 2/4] Overlap llama.cpp vision check with tokenization

---
 src/llama-cpp.ts         | 20 ++++++++++----------
 tests/llama-cpp.test.mjs | 32 ++++++++++++++++++++++++++++++--
 2 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/src/llama-cpp.ts b/src/llama-cpp.ts
index c93804e..7cb8aff 100644
--- a/src/llama-cpp.ts
+++ b/src/llama-cpp.ts
@@ -286,9 +286,6 @@ export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
       throw new TypeError("llama.cpp image inputs require addSpecialTokens to be false.");
     }
     signal?.throwIfAborted();
-    const imageSupport = hasImages
-      ? parseImageSupport(await get(fetchImpl, urls.props, headers, signal))
-      : undefined;
     const contents = [prompt, ...candidates.map((candidate) => prompt + candidate)];
     let requests = hasImages ? 1 : 0;
     const tokenizeIndividually = async (): Promise<number[][]> => {
@@ -316,10 +313,8 @@ export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
         }));
       return result;
     };
-    let encoded: number[][];
-    if (addSpecialTokens || batchUnsupported) {
-      encoded = await tokenizeIndividually();
-    } else {
+    const tokenize = async (): Promise<number[][]> => {
+      if (addSpecialTokens || batchUnsupported) return tokenizeIndividually();
       try {
         requests++;
         const content: (string | number)[] = [];
@@ -331,15 +326,20 @@ export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
           content, add_special: false, ...(model === undefined ? {} : { model }),
         }, signal, true);
         signal?.throwIfAborted();
-        encoded = parseBatchedTokenization(response, contents.length);
+        return parseBatchedTokenization(response, contents.length);
       } catch (error) {
         if (!(error instanceof UnsupportedBatchTokenization || error instanceof BatchTokenizationTooLarge)) {
           throw error;
         }
         if (error instanceof UnsupportedBatchTokenization) batchUnsupported = true;
-        encoded = await tokenizeIndividually();
+        return tokenizeIndividually();
       }
-    }
+    };
+    const [imageSupport, encoded] = hasImages
+      ? await Promise.all([
+        get(fetchImpl, urls.props, headers, signal).then(parseImageSupport), tokenize(),
+      ])
+      : [undefined, await tokenize()];
 
     const prefix = encoded[0]!;
     const { root: treeRoot, shared } = candidateTree(prefix, encoded.slice(1));
diff --git a/tests/llama-cpp.test.mjs b/tests/llama-cpp.test.mjs
index b98199d..85fc8c8 100644
--- a/tests/llama-cpp.test.mjs
+++ b/tests/llama-cpp.test.mjs
@@ -295,6 +295,34 @@ test("scores image labels through native multimodal completion", async () => {
   assert.equal(completions(f)[1].body.n_probs, 1);
 });
 
+test("tokenizes image choices while waiting for llama.cpp vision support", async () => {
+  const f = fixture();
+  const started = [];
+  let releaseProps;
+  const propsGate = new Promise((resolve) => { releaseProps = resolve; });
+  const fetch = async (url, init) => {
+    const path = new URL(url).pathname;
+    started.push(path);
+    if (path === "/props") await propsGate;
+    return f.fetch(url, init);
+  };
+  const choose = fromLlamaCpp({ baseURL: "http://localhost:8080", fetch });
+  const pending = choose({
+    ...choiceRequest,
+    images: [{ mediaType: "image/png", base64: "aW1hZ2U=" }],
+  });
+
+  try {
+    await new Promise((resolve) => setImmediate(resolve));
+    assert.ok(started.includes("/props"));
+    assert.ok(started.includes("/tokenize"));
+    assert.equal(started.includes("/completion"), false);
+  } finally {
+    releaseProps();
+  }
+  assert.equal((await pending).choice, "wait");
+});
+
 test("preserves a tokenization-boundary rollback for image labels", async () => {
   const pairs = (text) => {
     const bytes = new TextEncoder().encode(text);
@@ -401,7 +429,7 @@ test("rejects image inputs when llama.cpp does not advertise vision", async () =
     ...choiceRequest,
     images: [{ mediaType: "image/png", base64: "aW1hZ2U=" }],
   }), /does not advertise vision support/i);
-  assert.deepEqual(f.calls.map((call) => call.path), ["/props"]);
+  assert.equal(completions(f).length, 0);
 });
 
 test("rejects image inputs when llama.cpp omits the media marker", async () => {
@@ -412,7 +440,7 @@ test("rejects image inputs when llama.cpp omits the media marker", async () => {
     ...choiceRequest,
     images: [{ mediaType: "image/png", base64: "aW1hZ2U=" }],
   }), /multimodal media marker/i);
-  assert.deepEqual(f.calls.map((call) => call.path), ["/props"]);
+  assert.equal(completions(f).length, 0);
 });
 
 test("rejects an invalid llama.cpp detokenization response", async () => {

From 07ed3d983fafaed4854cc9529e8cbb3d0f4d190a Mon Sep 17 00:00:00 2001
From: NotXf1le <89696340+NotXf1le@users.noreply.github.com>
Date: Thu, 24 Sep 2026 00:19:58 +0200
Subject: [PATCH 3/4] Remove redundant image prefix retokenization

---
 src/llama-cpp.ts         | 11 -----------
 tests/llama-cpp.test.mjs | 12 ------------
 2 files changed, 23 deletions(-)

diff --git a/src/llama-cpp.ts b/src/llama-cpp.ts
index 7cb8aff..dfa39bb 100644
--- a/src/llama-cpp.ts
+++ b/src/llama-cpp.ts
@@ -367,17 +367,6 @@ export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
         throw new ScoringError("The formatted prompt contains llama.cpp's multimodal media marker.");
       }
 
-      const roundTripResponse = await post(fetchImpl, urls.tokenize, headers, {
-        content: detokenized, add_special: false,
-        ...(model === undefined ? {} : { model }),
-      }, signal);
-      requests++;
-      const roundTrip = parseTokenization(roundTripResponse);
-      if (roundTrip.length !== numericPrefix.length
-        || roundTrip.some((tokenId, index) => tokenId !== numericPrefix[index])) {
-        throw new ScoringError("llama.cpp could not preserve the image prompt token prefix.");
-      }
-
       const value = Object.freeze({
         prompt_string: `${images!.map(() => imageSupport!.marker).join("\n")}\n${detokenized}`,
         multimodal_data: Object.freeze(images!.map(({ base64 }) => base64)),
diff --git a/tests/llama-cpp.test.mjs b/tests/llama-cpp.test.mjs
index 85fc8c8..910452b 100644
--- a/tests/llama-cpp.test.mjs
+++ b/tests/llama-cpp.test.mjs
@@ -368,7 +368,6 @@ test("scores image labels that share an initial token", async () => {
     ["pA", [1, 100, 101]],
     ["pB", [1, 100, 102]],
     ["pC", [1, 200]],
-    ["px", [1, 100]],
   ]);
   const textByTokens = new Map([["1", "p"], ["1,100", "px"]]);
   const scores = new Map([[100, -0.2], [200, -2], [101, -0.3], [102, -1]]);
@@ -398,17 +397,6 @@ test("scores image labels that share an initial token", async () => {
   );
 });
 
-test("rejects an image prefix that does not survive a tokenization round trip", async () => {
-  const f = fixture({ detokenizer: () => "different text" });
-  const choose = fromLlamaCpp({ baseURL: "http://localhost:8080", fetch: f.fetch });
-
-  await assert.rejects(choose({
-    ...choiceRequest,
-    images: [{ mediaType: "image/png", base64: "aW1hZ2U=" }],
-  }), /could not preserve the image prompt token prefix/i);
-  assert.equal(completions(f).length, 0);
-});
-
 test("rejects a formatted prompt containing llama.cpp's media marker", async () => {
   const marker = "<__media_test__>";
   const f = fixture({ detokenizer: () => `prompt ${marker}` });

From e7975f4de6650a873a2918ba7744e73f1a50b03b Mon Sep 17 00:00:00 2001
From: NotXf1le <89696340+NotXf1le@users.noreply.github.com>
Date: Thu, 24 Sep 2026 08:43:30 +0200
Subject: [PATCH 4/4] Make missing llama.cpp logprob probes optional

---
 README.md                |  2 ++
 src/llama-cpp.ts         | 15 +++++++++++++--
 tests/llama-cpp.test.mjs | 36 +++++++++++++++++++++++++++++++++++-
 3 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index b854407..09c611d 100644
--- a/README.md
+++ b/README.md
@@ -79,6 +79,8 @@ console.log(decision.distribution); // { yes: ..., no: ... }
 
 The llama.cpp backend requires its native `/tokenize` and `/completion` endpoints. `minimal-prefix` is available only with this backend.
 
+Set `probeMissingLogprobs: true` to score low-probability choices outside llama.cpp's returned `top_logprobs`.
+
 The library has no telemetry.
 
 ## Ollama
diff --git a/src/llama-cpp.ts b/src/llama-cpp.ts
index dfa39bb..ec3468b 100644
--- a/src/llama-cpp.ts
+++ b/src/llama-cpp.ts
@@ -14,6 +14,8 @@ export interface LlamaCppOptions extends ChooserOptions {
   readonly fetch?: typeof globalThis.fetch;
   /** Controls add_special for llama.cpp tokenization. Defaults to false; image inputs require false. */
   readonly addSpecialTokens?: boolean;
+  /** Probe choices missing from the first top-logprob list. Defaults to false. */
+  readonly probeMissingLogprobs?: boolean;
 }
 
 interface Endpoints {
@@ -263,12 +265,16 @@ function parseProbe(value: unknown, expectedPromptTokens: number | null,
 
 export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
   if (!isRecord(options)) throw new TypeError("options must be an object.");
-  const { baseURL, model, mode = "labels", addSpecialTokens = false, formatPrompt } = options;
+  const { baseURL, model, mode = "labels", addSpecialTokens = false,
+    probeMissingLogprobs = false, formatPrompt } = options;
   if (model !== undefined) requireText(model, "model");
   if (mode !== "labels" && mode !== "minimal-prefix") {
     throw new TypeError("mode must be labels or minimal-prefix.");
   }
   if (typeof addSpecialTokens !== "boolean") throw new TypeError("addSpecialTokens must be a boolean.");
+  if (typeof probeMissingLogprobs !== "boolean") {
+    throw new TypeError("probeMissingLogprobs must be a boolean.");
+  }
   if (options.fetch !== undefined && typeof options.fetch !== "function") {
     throw new TypeError("fetch must be a function.");
   }
@@ -396,6 +402,7 @@ export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
       const promptValue = hasImages ? await materializeImagePrompt(numericPrefix) : numericPrefix;
       for (const [targetTokenId] of children) {
         if (siblingLogprobs.has(targetTokenId)) continue;
+        if (!probeMissingLogprobs && siblingLogprobs.size > 0) break;
         const collectSiblings = siblingLogprobs.size === 0 && children.length > 1;
         signal?.throwIfAborted();
         const response = await post(fetchImpl, urls.completion, headers, {
@@ -437,7 +444,11 @@ export function fromLlamaCpp(options: LlamaCppOptions): Chooser {
       for (const [targetTokenId, node] of children) {
         const logprob = siblingLogprobs.get(targetTokenId);
         if (logprob === undefined) {
-          throw new ScoringError("A candidate branch is missing its raw log probability.");
+          if (probeMissingLogprobs) {
+            throw new ScoringError("A candidate branch is missing its raw log probability.");
+          }
+          for (const index of groups.get(targetTokenId)!) scores[index] = -Infinity;
+          continue;
         }
         const value = branch.score + logprob;
         if (!Number.isFinite(value)) {
diff --git a/tests/llama-cpp.test.mjs b/tests/llama-cpp.test.mjs
index 910452b..23981c3 100644
--- a/tests/llama-cpp.test.mjs
+++ b/tests/llama-cpp.test.mjs
@@ -257,6 +257,7 @@ test("scores image labels through native multimodal completion", async () => {
   const choose = fromLlamaCpp({
     baseURL: "http://localhost:8080/v1",
     model: "vision/model",
+    probeMissingLogprobs: true,
     headers: { authorization: "Bearer test" },
     fetch: f.fetch,
   });
@@ -378,6 +379,7 @@ test("scores image labels that share an initial token", async () => {
   });
   const choose = fromLlamaCpp({
     baseURL: "http://localhost:8080",
+    probeMissingLogprobs: true,
     fetch: f.fetch,
     formatPrompt: ({ context }) => context,
   });
@@ -468,7 +470,7 @@ test("rejects special-token insertion for llama.cpp image inputs", async () => {
   assert.equal(f.calls.length, 0);
 });
 
-test("scores candidates separately when top logprobs contain none of them", async () => {
+test("assigns zero probability to choices missing from the first top logprobs", async () => {
   const scores = new Map([[encode("A")[0], -0.3], [encode("B")[0], -1.4]]);
   const f = fixture({
     logprob: (id) => scores.get(id) ?? -10,
@@ -477,6 +479,23 @@ test("scores candidates separately when top logprobs contain none of them", asyn
 
   const decision = await fromLlamaCpp({ baseURL: "http://localhost:8080", fetch: f.fetch })(choiceRequest);
 
+  assert.deepEqual(decision.scores, { wait: -0.3, deploy: -Infinity });
+  assert.deepEqual(decision.distribution, { wait: 1, deploy: 0 });
+  assert.equal(completions(f).length, 1);
+  assert.equal(decision.usage.requests, f.calls.length);
+});
+
+test("probes choices missing from the first top logprobs when requested", async () => {
+  const scores = new Map([[encode("A")[0], -0.3], [encode("B")[0], -1.4]]);
+  const f = fixture({
+    logprob: (id) => scores.get(id) ?? -10,
+    topTokenIds: () => [999],
+  });
+
+  const decision = await fromLlamaCpp({
+    baseURL: "http://localhost:8080", probeMissingLogprobs: true, fetch: f.fetch,
+  })(choiceRequest);
+
   assert.equal(decision.choice, "wait");
   assert.deepEqual(decision.scores, { wait: -0.3, deploy: -1.4 });
   assert.equal(completions(f).length, 2);
@@ -533,6 +552,21 @@ test("continues scoring a group that separates at a deeper branch", async () =>
     completions(f)[0].body.prompt.length + 1);
 });
 
+test("assigns zero probability to every key below a missing minimal-prefix branch", async () => {
+  const a = encode("a")[0];
+  const f = fixture({ topTokenIds: () => [a] });
+  const choose = fromLlamaCpp({
+    baseURL: "http://localhost:8080", mode: "minimal-prefix", fetch: f.fetch,
+  });
+
+  const decision = await choose({
+    context: "Choose a key.", question: "Which key?", choices: { a: "A", ba: "BA", bb: "BB" },
+  });
+
+  assert.deepEqual(decision.distribution, { a: 1, ba: 0, bb: 0 });
+  assert.equal(completions(f).length, 1);
+});
+
 test("reports a tokenization-boundary rollback", async () => {
   const pairs = (text, special = false) => {
     const bytes = new TextEncoder().encode(text);