diff --git a/CHANGELOG.md b/CHANGELOG.md index e6eef40..6370546 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- Added an exact `/v1/models` check to the SemIf llama.cpp benchmark, with + `--skip-model-check` for unverified runs. + ## 0.5.0 - 2026-09-20 - Added label scoring through OpenRouter with `choosekit/openrouter`. diff --git a/benchmarks/README.md b/benchmarks/README.md index e140136..ac65d0f 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -91,7 +91,15 @@ node benchmarks/run-semif.mjs \ --base-url http://127.0.0.1:11434/ \ --model qwen3.8-27b-text-64k \ --output benchmarks/results/semif-qwen-labels.json +``` + +Before running, the script requires the selected model ID to appear in `GET /v1/models`. This prevents +a report from being labelled with a model the server did not expose. Use `--skip-model-check` to bypass +the check; the report then records `modelChecked: false` in its `runtime` block. +Run the Jev comparison with an OpenRouter API key: + +```sh OPENROUTER_API_KEY=... node benchmarks/run-semif-openrouter-jev.mjs \ --model typesafe/jev-1.13 \ --output benchmarks/results/semif-jev-1.13.json diff --git a/benchmarks/run-semif.mjs b/benchmarks/run-semif.mjs index 3e49a8e..460d9df 100644 --- a/benchmarks/run-semif.mjs +++ b/benchmarks/run-semif.mjs @@ -89,6 +89,48 @@ function summarize(results, startedAt) { }; } +/** llama.cpp serves /v1/models beside /completion; mirror the adapter's base-URL handling. */ +function modelsURL(baseURL) { + const url = new URL(baseURL); + if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password + || url.search || url.hash) { + throw new TypeError("baseURL must be an HTTP(S) URL without credentials, query, or fragment."); + } + const path = url.pathname.replace(/\/v1\/?$/, "").replace(/\/$/, ""); + url.pathname = `${path}/v1/models`; + return url.href; +} + +async function servedModelIds(url) { + const response = await fetch(url, { signal: AbortSignal.timeout(30_000) }); + if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`); + const body = await response.json(); + const entries = Array.isArray(body?.data) ? body.data : []; + const ids = entries.map((entry) => entry?.id).filter((id) => typeof id === "string" && id.length > 0); + if (ids.length === 0) throw new Error("the response listed no models"); + return ids; +} + +/** Verify that the report's model ID is listed by the server before inference. */ +async function checkModel(baseURL, model) { + const url = modelsURL(baseURL); + let ids; + try { + ids = await servedModelIds(url); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`Could not list the models at ${url}: ${detail}.` + + " Ensure the server exposes /v1/models, or pass --skip-model-check" + + " to record the run as unverified."); + } + if (!ids.includes(model)) { + throw new Error(`The server at ${url} does not list "${model}".` + + ` Available models: ${ids.map((id) => `"${id}"`).join(", ")}.` + + " Pass --model with one of the IDs above, or --skip-model-check" + + " to record the run as unverified."); + } +} + const input = option("--input", "benchmarks/data/semif-authored144.jsonl"); const mode = option("--mode", "labels"); if (mode !== "labels" && mode !== "minimal-prefix") { @@ -98,6 +140,7 @@ const output = option("--output", `benchmarks/results/semif-qwen3.8-27b-${mode}. const baseURL = option("--base-url", process.env.LLAMA_CPP_BASE_URL ?? "http://127.0.0.1:11434/"); const model = option("--model", process.env.LLAMA_CPP_MODEL ?? "qwen3.8-27b-text-64k"); +const skipModelCheck = process.argv.includes("--skip-model-check"); const rawLimit = option("--limit", undefined); const limit = rawLimit === undefined ? undefined : Number.parseInt(rawLimit, 10); @@ -111,6 +154,11 @@ const allRows = source.toString("utf8").trim().split(/\r?\n/).map((line) => JSON if (allRows.length !== 144) throw new Error(`Expected 144 SemIf rows, received ${allRows.length}.`); const rows = limit === undefined ? allRows : allRows.slice(0, limit); mkdirSync(dirname(output), { recursive: true }); +if (skipModelCheck) console.warn(`--skip-model-check: recording unverified model ID "${model}".`); +else { + await checkModel(baseURL, model); + console.log(`Model ID confirmed in the server catalog: ${model}`); +} const choose = fromLlamaCpp({ baseURL, model, mode }); const results = []; const startedAt = performance.now(); @@ -172,7 +220,14 @@ for (let index = 0; index < rows.length; index++) { totalRows: allRows.length, selectedRows: rows.length, }, - runtime: { baseURL, model, mode, adapter: "choosekit/llama-cpp", packageVersion: "0.5.0" }, + runtime: { + baseURL, + model, + modelChecked: !skipModelCheck, + mode, + adapter: "choosekit/llama-cpp", + packageVersion: "0.5.0", + }, interpretation: mode === "labels" ? "Package A/B/C label prompt and distinguishing-token likelihoods." : "Package original-key prompt and minimal distinguishing-prefix likelihoods.", diff --git a/tests/benchmark.test.mjs b/tests/benchmark.test.mjs index f9e3f50..a664932 100644 --- a/tests/benchmark.test.mjs +++ b/tests/benchmark.test.mjs @@ -58,6 +58,66 @@ test("llama.cpp benchmark creates its output directory before inference", () => assert.equal(report.results.length, 1); assert.equal(report.summary.errors, 1); assert.match(report.results[0].error, /Intentional benchmark test failure/); + assert.equal(report.runtime.modelChecked, true); +}); + +test("llama.cpp benchmark refuses a model the server does not serve", () => { + const output = temporaryPath("qwen.json"); + const result = run("run-semif.mjs", [ + "--input", input, "--output", output, "--limit", "1", + ], { + CHOOSEKIT_EXPECT_OUTPUT_PARENT: dirname(output), + CHOOSEKIT_SERVED_MODELS: "/gguf/LFM2.5-1.2B-Instruct-Q8_0.gguf", + }); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /does not list "qwen3\.8-27b-text-64k"/); + assert.match(result.stderr, /Available models: "\/gguf\/LFM2\.5-1\.2B-Instruct-Q8_0\.gguf"/); + assert.throws(() => readFileSync(output)); +}); + +test("llama.cpp benchmark explains an unreachable model list", () => { + const output = temporaryPath("qwen.json"); + const result = run("run-semif.mjs", [ + "--input", input, "--output", output, "--limit", "1", + ], { CHOOSEKIT_EXPECT_OUTPUT_PARENT: dirname(output), CHOOSEKIT_MODELS_STATUS: "404" }); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Could not list the models at .*\/v1\/models: HTTP 404/); + assert.match(result.stderr, /--skip-model-check/); +}); + +test("llama.cpp benchmark rejects unsafe base URLs before model discovery", () => { + const output = temporaryPath("qwen.json"); + const result = run("run-semif.mjs", [ + "--input", input, + "--output", output, + "--limit", "1", + "--base-url", "http://user:benchmark-secret@127.0.0.1:11434/?token=query-secret", + ], { + CHOOSEKIT_EXPECT_OUTPUT_PARENT: dirname(output), + CHOOSEKIT_MODELS_STATUS: "404", + }); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, + /TypeError: baseURL must be an HTTP\(S\) URL without credentials, query, or fragment\./); + assert.doesNotMatch(result.stderr, /benchmark-secret|query-secret/); + assert.throws(() => readFileSync(output)); +}); + +test("llama.cpp benchmark records an unchecked run as unchecked", () => { + const output = temporaryPath("qwen.json"); + const result = run("run-semif.mjs", [ + "--input", input, "--output", output, "--limit", "1", "--skip-model-check", + ], { + CHOOSEKIT_EXPECT_OUTPUT_PARENT: dirname(output), + CHOOSEKIT_SERVED_MODELS: "/gguf/LFM2.5-1.2B-Instruct-Q8_0.gguf", + }); + + assert.equal(result.status, 0, result.stderr); + const report = JSON.parse(readFileSync(output, "utf8")); + assert.equal(report.runtime.modelChecked, false); }); test("OpenRouter benchmark creates its output directory before inference", () => { diff --git a/tests/fixtures/benchmark-fetch.mjs b/tests/fixtures/benchmark-fetch.mjs index bbbb01b..90cae9c 100644 --- a/tests/fixtures/benchmark-fetch.mjs +++ b/tests/fixtures/benchmark-fetch.mjs @@ -1,6 +1,23 @@ import { existsSync } from "node:fs"; -globalThis.fetch = async () => { +/** The llama.cpp benchmark lists /v1/models before inference; everything else fails on purpose. */ +function servedModels() { + const ids = (process.env.CHOOSEKIT_SERVED_MODELS ?? "qwen3.8-27b-text-64k") + .split(",").filter((id) => id.length > 0); + return new Response(JSON.stringify({ object: "list", data: ids.map((id) => ({ id, object: "model" })) }), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +globalThis.fetch = async (input) => { + const url = typeof input === "string" ? input : input.url; + if (new URL(url).pathname.endsWith("/v1/models")) { + if (process.env.CHOOSEKIT_MODELS_STATUS) { + return new Response("model list unavailable", { status: Number(process.env.CHOOSEKIT_MODELS_STATUS) }); + } + return servedModels(); + } const expectedParent = process.env.CHOOSEKIT_EXPECT_OUTPUT_PARENT; if (!expectedParent || !existsSync(expectedParent)) { throw new Error("Benchmark output directory was not created before the first request.");