benchmarks: confirm --model against the server before running (#10)

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Unreleased
+
+- Added an exact `/v1/models` check to the SemIf llama.cpp benchmark, with
+  `--skip-model-check` for unverified runs.
+
 ## 0.5.0 - 2026-09-20
 
 - Added label scoring through OpenRouter with `choosekit/openrouter`.

benchmarks/README.md

@@ -91,7 +91,15 @@ node benchmarks/run-semif.mjs \
   --base-url http://127.0.0.1:11434/ \
   --model qwen3.8-27b-text-64k \
   --output benchmarks/results/semif-qwen-labels.json
+```
+
+Before running, the script requires the selected model ID to appear in `GET /v1/models`. This prevents
+a report from being labelled with a model the server did not expose. Use `--skip-model-check` to bypass
+the check; the report then records `modelChecked: false` in its `runtime` block.
 
+Run the Jev comparison with an OpenRouter API key:
+
+```sh
 OPENROUTER_API_KEY=... node benchmarks/run-semif-openrouter-jev.mjs \
   --model typesafe/jev-1.13 \
   --output benchmarks/results/semif-jev-1.13.json

benchmarks/run-semif.mjs

@@ -89,6 +89,48 @@ function summarize(results, startedAt) {
   };
 }
 
+/** llama.cpp serves /v1/models beside /completion; mirror the adapter's base-URL handling. */
+function modelsURL(baseURL) {
+  const url = new URL(baseURL);
+  if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password
+    || url.search || url.hash) {
+    throw new TypeError("baseURL must be an HTTP(S) URL without credentials, query, or fragment.");
+  }
+  const path = url.pathname.replace(/\/v1\/?$/, "").replace(/\/$/, "");
+  url.pathname = `${path}/v1/models`;
+  return url.href;
+}
+
+async function servedModelIds(url) {
+  const response = await fetch(url, { signal: AbortSignal.timeout(30_000) });
+  if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`);
+  const body = await response.json();
+  const entries = Array.isArray(body?.data) ? body.data : [];
+  const ids = entries.map((entry) => entry?.id).filter((id) => typeof id === "string" && id.length > 0);
+  if (ids.length === 0) throw new Error("the response listed no models");
+  return ids;
+}
+
+/** Verify that the report's model ID is listed by the server before inference. */
+async function checkModel(baseURL, model) {
+  const url = modelsURL(baseURL);
+  let ids;
+  try {
+    ids = await servedModelIds(url);
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new Error(`Could not list the models at ${url}: ${detail}.`
+      + " Ensure the server exposes /v1/models, or pass --skip-model-check"
+      + " to record the run as unverified.");
+  }
+  if (!ids.includes(model)) {
+    throw new Error(`The server at ${url} does not list "${model}".`
+      + ` Available models: ${ids.map((id) => `"${id}"`).join(", ")}.`
+      + " Pass --model with one of the IDs above, or --skip-model-check"
+      + " to record the run as unverified.");
+  }
+}
+
 const input = option("--input", "benchmarks/data/semif-authored144.jsonl");
 const mode = option("--mode", "labels");
 if (mode !== "labels" && mode !== "minimal-prefix") {
@@ -98,6 +140,7 @@ const output = option("--output", `benchmarks/results/semif-qwen3.8-27b-${mode}.
 const baseURL = option("--base-url", process.env.LLAMA_CPP_BASE_URL
   ?? "http://127.0.0.1:11434/");
 const model = option("--model", process.env.LLAMA_CPP_MODEL ?? "qwen3.8-27b-text-64k");
+const skipModelCheck = process.argv.includes("--skip-model-check");
 const rawLimit = option("--limit", undefined);
 const limit = rawLimit === undefined ? undefined : Number.parseInt(rawLimit, 10);
 
@@ -111,6 +154,11 @@ const allRows = source.toString("utf8").trim().split(/\r?\n/).map((line) => JSON
 if (allRows.length !== 144) throw new Error(`Expected 144 SemIf rows, received ${allRows.length}.`);
 const rows = limit === undefined ? allRows : allRows.slice(0, limit);
 mkdirSync(dirname(output), { recursive: true });
+if (skipModelCheck) console.warn(`--skip-model-check: recording unverified model ID "${model}".`);
+else {
+  await checkModel(baseURL, model);
+  console.log(`Model ID confirmed in the server catalog: ${model}`);
+}
 const choose = fromLlamaCpp({ baseURL, model, mode });
 const results = [];
 const startedAt = performance.now();
@@ -172,7 +220,14 @@ for (let index = 0; index < rows.length; index++) {
       totalRows: allRows.length,
       selectedRows: rows.length,
     },
-    runtime: { baseURL, model, mode, adapter: "choosekit/llama-cpp", packageVersion: "0.5.0" },
+    runtime: {
+      baseURL,
+      model,
+      modelChecked: !skipModelCheck,
+      mode,
+      adapter: "choosekit/llama-cpp",
+      packageVersion: "0.5.0",
+    },
     interpretation: mode === "labels"
       ? "Package A/B/C label prompt and distinguishing-token likelihoods."
       : "Package original-key prompt and minimal distinguishing-prefix likelihoods.",

tests/benchmark.test.mjs

@@ -58,6 +58,66 @@ test("llama.cpp benchmark creates its output directory before inference", () =>
   assert.equal(report.results.length, 1);
   assert.equal(report.summary.errors, 1);
   assert.match(report.results[0].error, /Intentional benchmark test failure/);
+  assert.equal(report.runtime.modelChecked, true);
+});
+
+test("llama.cpp benchmark refuses a model the server does not serve", () => {
+  const output = temporaryPath("qwen.json");
+  const result = run("run-semif.mjs", [
+    "--input", input, "--output", output, "--limit", "1",
+  ], {
+    CHOOSEKIT_EXPECT_OUTPUT_PARENT: dirname(output),
+    CHOOSEKIT_SERVED_MODELS: "/gguf/LFM2.5-1.2B-Instruct-Q8_0.gguf",
+  });
+
+  assert.notEqual(result.status, 0);
+  assert.match(result.stderr, /does not list "qwen3\.8-27b-text-64k"/);
+  assert.match(result.stderr, /Available models: "\/gguf\/LFM2\.5-1\.2B-Instruct-Q8_0\.gguf"/);
+  assert.throws(() => readFileSync(output));
+});
+
+test("llama.cpp benchmark explains an unreachable model list", () => {
+  const output = temporaryPath("qwen.json");
+  const result = run("run-semif.mjs", [
+    "--input", input, "--output", output, "--limit", "1",
+  ], { CHOOSEKIT_EXPECT_OUTPUT_PARENT: dirname(output), CHOOSEKIT_MODELS_STATUS: "404" });
+
+  assert.notEqual(result.status, 0);
+  assert.match(result.stderr, /Could not list the models at .*\/v1\/models: HTTP 404/);
+  assert.match(result.stderr, /--skip-model-check/);
+});
+
+test("llama.cpp benchmark rejects unsafe base URLs before model discovery", () => {
+  const output = temporaryPath("qwen.json");
+  const result = run("run-semif.mjs", [
+    "--input", input,
+    "--output", output,
+    "--limit", "1",
+    "--base-url", "http://user:benchmark-secret@127.0.0.1:11434/?token=query-secret",
+  ], {
+    CHOOSEKIT_EXPECT_OUTPUT_PARENT: dirname(output),
+    CHOOSEKIT_MODELS_STATUS: "404",
+  });
+
+  assert.notEqual(result.status, 0);
+  assert.match(result.stderr,
+    /TypeError: baseURL must be an HTTP\(S\) URL without credentials, query, or fragment\./);
+  assert.doesNotMatch(result.stderr, /benchmark-secret|query-secret/);
+  assert.throws(() => readFileSync(output));
+});
+
+test("llama.cpp benchmark records an unchecked run as unchecked", () => {
+  const output = temporaryPath("qwen.json");
+  const result = run("run-semif.mjs", [
+    "--input", input, "--output", output, "--limit", "1", "--skip-model-check",
+  ], {
+    CHOOSEKIT_EXPECT_OUTPUT_PARENT: dirname(output),
+    CHOOSEKIT_SERVED_MODELS: "/gguf/LFM2.5-1.2B-Instruct-Q8_0.gguf",
+  });
+
+  assert.equal(result.status, 0, result.stderr);
+  const report = JSON.parse(readFileSync(output, "utf8"));
+  assert.equal(report.runtime.modelChecked, false);
 });
 
 test("OpenRouter benchmark creates its output directory before inference", () => {

tests/fixtures/benchmark-fetch.mjs

@@ -1,6 +1,23 @@
 import { existsSync } from "node:fs";
 
-globalThis.fetch = async () => {
+/** The llama.cpp benchmark lists /v1/models before inference; everything else fails on purpose. */
+function servedModels() {
+  const ids = (process.env.CHOOSEKIT_SERVED_MODELS ?? "qwen3.8-27b-text-64k")
+    .split(",").filter((id) => id.length > 0);
+  return new Response(JSON.stringify({ object: "list", data: ids.map((id) => ({ id, object: "model" })) }), {
+    status: 200,
+    headers: { "content-type": "application/json" },
+  });
+}
+
+globalThis.fetch = async (input) => {
+  const url = typeof input === "string" ? input : input.url;
+  if (new URL(url).pathname.endsWith("/v1/models")) {
+    if (process.env.CHOOSEKIT_MODELS_STATUS) {
+      return new Response("model list unavailable", { status: Number(process.env.CHOOSEKIT_MODELS_STATUS) });
+    }
+    return servedModels();
+  }
   const expectedParent = process.env.CHOOSEKIT_EXPECT_OUTPUT_PARENT;
   if (!expectedParent || !existsSync(expectedParent)) {
     throw new Error("Benchmark output directory was not created before the first request.");