diff --git a/CHANGELOG.md b/CHANGELOG.md index be063dc..5962e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- Added the MCP `chooseMany` tool for independent named questions with shared context and optional images. + ## 0.8.0 - 2026-10-10 - Added `chooseMany()` for independent questions with common context and optional images. diff --git a/README.md b/README.md index 7388c88..7683193 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The project was inspired by [Jev and the System One model interface](https://typ ## MCP server -[`choosekit-mcp`](packages/choosekit-mcp/README.md) exposes choosekit through llama.cpp, Ollama, or OpenRouter as a read-only stdio tool for Claude Code, Codex, and OpenCode. Select the backend and configure it with environment variables when starting the MCP server. +[`choosekit-mcp`](packages/choosekit-mcp/README.md) provides read-only `choose` and `chooseMany` MCP tools through llama.cpp, Ollama, or OpenRouter for Claude Code, Codex, and OpenCode. Select the backend and configure it with environment variables when starting the MCP server. ## llama.cpp diff --git a/packages/choosekit-mcp/README.md b/packages/choosekit-mcp/README.md index 50b2ca6..5f28a10 100644 --- a/packages/choosekit-mcp/README.md +++ b/packages/choosekit-mcp/README.md @@ -1,6 +1,6 @@ # choosekit-mcp -`choosekit-mcp` exposes [choosekit](https://github.com/NotXf1le/choosekit) as a read-only MCP tool backed by llama.cpp, Ollama, or OpenRouter. +`choosekit-mcp` exposes [choosekit](https://github.com/NotXf1le/choosekit) through read-only MCP tools backed by llama.cpp, Ollama, or OpenRouter. The server uses stdio. It returns a decision and probability distribution. Execution of the selected action remains with the MCP client. @@ -60,9 +60,11 @@ codex mcp add choosekit --env CHOOSEKIT_BACKEND=openrouter --env OPENROUTER_API_ On native Windows, launch `npx` through `cmd` in any example above: replace `-- npx -y choosekit-mcp` with `-- cmd /c npx -y choosekit-mcp`. -## Tool +## Tools -The server exposes one tool named `choose`: +### choose + +Score a question against the supplied choices: ```json { @@ -76,7 +78,37 @@ The server exposes one tool named `choose`: } ``` -To score images, add `imagePaths` in `labels` mode: +### chooseMany + +Ask independent named questions about the same context: + +```json +{ + "context": "A payout failed three days in a row. Payroll is due today.", + "questions": { + "route": { + "question": "Which team should handle this?", + "choices": { + "billing": "Payments, payouts, invoices, and refunds", + "technical": "Bugs, outages, integrations, and API errors" + } + }, + "urgency": { + "question": "How urgent is this?", + "choices": { + "low": "The request can wait", + "high": "The request needs immediate attention" + } + } + } +} +``` + +The result maps each question name to its decision, including separate token counts when available. Choice limits apply to each question. + +### Images + +To score images, add `imagePaths` in `labels` mode. Both tools accept this field alongside `context`; `chooseMany` shares the images across all questions. ```json { diff --git a/packages/choosekit-mcp/package-lock.json b/packages/choosekit-mcp/package-lock.json index fbb4abe..a85b25b 100644 --- a/packages/choosekit-mcp/package-lock.json +++ b/packages/choosekit-mcp/package-lock.json @@ -1,16 +1,16 @@ { "name": "choosekit-mcp", - "version": "0.4.0", + "version": "0.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "choosekit-mcp", - "version": "0.4.0", + "version": "0.5.0", "license": "Apache-2.0", "dependencies": { "@modelcontextprotocol/server": "2.0.0", - "choosekit": "^0.7.0", + "choosekit": "^0.8.0", "zod": "^4.6.4" }, "bin": { @@ -60,9 +60,9 @@ } }, "node_modules/choosekit": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/choosekit/-/choosekit-0.7.0.tgz", - "integrity": "sha512-BbJt3psfsi9JRUvdfB2qUT6huECmvjTgQ+A+b17DnOmoJg4hdbr4eYJOYS9HobugRrRXXrAkC9ij6Z3vvWBbpA==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/choosekit/-/choosekit-0.8.0.tgz", + "integrity": "sha512-3+E92te0MYmQ8H49DR4t2AFnM7L7o70bXHh6zE0kHpLfRyFoiss+zv6rddc0Z8gYSa9dkOqqx+9IdUn3ZtiXEw==", "license": "Apache-2.0", "engines": { "node": ">=20" diff --git a/packages/choosekit-mcp/package.json b/packages/choosekit-mcp/package.json index 700ce76..1238f37 100644 --- a/packages/choosekit-mcp/package.json +++ b/packages/choosekit-mcp/package.json @@ -1,6 +1,6 @@ { "name": "choosekit-mcp", - "version": "0.4.0", + "version": "0.5.0", "description": "Use choosekit as an MCP server.", "license": "Apache-2.0", "author": { @@ -37,7 +37,7 @@ }, "dependencies": { "@modelcontextprotocol/server": "2.0.0", - "choosekit": "^0.7.0", + "choosekit": "^0.8.0", "zod": "^4.6.4" }, "devDependencies": { diff --git a/packages/choosekit-mcp/src/server.ts b/packages/choosekit-mcp/src/server.ts index 9f46dc1..d1c5873 100644 --- a/packages/choosekit-mcp/src/server.ts +++ b/packages/choosekit-mcp/src/server.ts @@ -1,5 +1,5 @@ import { McpServer } from "@modelcontextprotocol/server"; -import { ScoringError, type Chooser, type Decision, type Usage } from "choosekit"; +import { chooseMany, ScoringError, type Chooser, type Decision } from "choosekit"; import { createRequire } from "node:module"; import { z } from "zod"; import { ImageLoadError, type ImageLoader } from "./images.js"; @@ -31,7 +31,7 @@ const decisionSchema = z.object({ usage: usageSchema.optional(), }).strict(); -function inputSchema(backend: "llama-cpp" | "ollama" | "openrouter", mode: ChoiceMode, +function inputSchemas(backend: "llama-cpp" | "ollama" | "openrouter", mode: ChoiceMode, imagesEnabled: boolean) { const choicesSchema = z.fromJSONSchema({ type: "object", @@ -49,17 +49,28 @@ function inputSchema(backend: "llama-cpp" | "ollama" | "openrouter", mode: Choic : mode === "labels" ? { maxProperties: 26 } : {}), }) as z.ZodType>; - return z.object({ - context: z.string().describe("The context to use when making the decision."), - question: z.string() - .refine((question) => question.trim().length > 0, "The question must not be empty."), + const questionSchema = z.object({ + question: z.string().regex(/\S/, "The question must not be empty."), choices: choicesSchema, + }).strict(); + const sharedFields = { + context: z.string().describe("The context to use when making the decision."), ...(mode === "labels" && imagesEnabled ? { imagePaths: z.array(z.string().min(1)).min(1) .describe("Image file paths inside the configured image root, in display order.") .optional(), } : {}), - }).strict(); + }; + const questionsSchema = z.fromJSONSchema({ + type: "object", + propertyNames: { type: "string", pattern: "^(?!__proto__$).+$" }, + minProperties: 1, + additionalProperties: z.toJSONSchema(questionSchema), + }) as z.ZodType>>; + return { + choose: z.object({ ...sharedFields, ...questionSchema.shape }).strict(), + chooseMany: z.object({ ...sharedFields, questions: questionsSchema }).strict(), + }; } function toolDescription(mode: ChoiceMode): string { @@ -100,6 +111,18 @@ Readonly> { [key, score === -Infinity ? null : score])); } +function wireDecision(decision: Decision) { + return { + choice: decision.choice, + distribution: decision.distribution, + scores: wireScores(decision.scores), + margin: decision.margin, + entropy: decision.entropy, + boundaryTokens: decision.boundaryTokens, + ...(decision.usage === undefined ? {} : { usage: decision.usage }), + }; +} + export function buildServer(chooser: Chooser, options: ServerOptions = {}): McpServer { if (typeof chooser !== "function") throw new TypeError("chooser must be a function."); const backend = options.backend ?? "llama-cpp"; @@ -112,11 +135,12 @@ export function buildServer(chooser: Chooser, options: ServerOptions = {}): McpS } const server = new McpServer({ name: "choosekit-mcp", version: packageVersion }); + const schemas = inputSchemas(backend, mode, options.imageLoader !== undefined); server.registerTool( "choose", { description: toolDescription(mode), - inputSchema: inputSchema(backend, mode, options.imageLoader !== undefined), + inputSchema: schemas.choose, outputSchema: decisionSchema, annotations: { readOnlyHint: true, @@ -135,23 +159,40 @@ export function buildServer(chooser: Chooser, options: ServerOptions = {}): McpS ...(images === undefined ? {} : { images }), signal: ctx.mcpReq.signal, }) as Decision; - const structuredContent: { - choice: string; - distribution: Readonly>; - scores: Readonly>; - margin: number; - entropy: number; - boundaryTokens: number; - usage?: Usage; - } = { - choice: decision.choice, - distribution: decision.distribution, - scores: wireScores(decision.scores), - margin: decision.margin, - entropy: decision.entropy, - boundaryTokens: decision.boundaryTokens, - ...(decision.usage === undefined ? {} : { usage: decision.usage }), + const structuredContent = wireDecision(decision); + return { + content: [{ type: "text", text: JSON.stringify(structuredContent) }], + structuredContent, }; + } catch (error) { + return errorResult(error, ctx.mcpReq.signal); + } + }, + ); + server.registerTool( + "chooseMany", + { + description: "Answer independent named questions about the same context. Each question is scored separately. " + toolDescription(mode), + inputSchema: schemas.chooseMany, + outputSchema: z.record(z.string(), decisionSchema), + annotations: { + readOnlyHint: true, + openWorldHint: backend === "openrouter", + }, + }, + async ({ context, questions, imagePaths }, ctx) => { + try { + const images = imagePaths === undefined + ? undefined + : await options.imageLoader!(imagePaths as readonly string[], ctx.mcpReq.signal); + const decisions = await chooseMany(chooser, { + context, + questions, + ...(images === undefined ? {} : { images }), + signal: ctx.mcpReq.signal, + }); + const structuredContent = Object.fromEntries(Object.entries(decisions) + .map(([name, decision]) => [name, wireDecision(decision)])); return { content: [{ type: "text", text: JSON.stringify(structuredContent) }], structuredContent, diff --git a/packages/choosekit-mcp/tests/cli.test.mjs b/packages/choosekit-mcp/tests/cli.test.mjs index 89ffa27..5b03ce7 100644 --- a/packages/choosekit-mcp/tests/cli.test.mjs +++ b/packages/choosekit-mcp/tests/cli.test.mjs @@ -179,7 +179,7 @@ test("defaults to llama.cpp and lists tools without contacting it", async (t) => const response = await process_.rpc("tools/list"); - assert.deepEqual(response.result.tools.map((tool) => tool.name), ["choose"]); + assert.deepEqual(response.result.tools.map((tool) => tool.name), ["choose", "chooseMany"]); assert.ok(process_.lines.length >= 2); for (const line of process_.lines) assert.doesNotThrow(() => JSON.parse(line)); }); @@ -262,6 +262,23 @@ test("serves an Ollama image choice through its configured chat endpoint", async assert.equal(requests[0].url, "/api/chat"); assert.equal(requests[0].body.model, "fixture-model"); assert.deepEqual(requests[0].body.messages[0].images, [image.toString("base64")]); + const many = await process_.rpc("tools/call", { + name: "chooseMany", + arguments: { + context: request.context, + imagePaths: ["example.png"], + questions: { + route: { question: request.question, choices: request.choices }, + urgency: { question: "How urgent is this?", choices: { low: "Can wait", high: "Needs immediate attention" } }, + }, + }, + }); + assert.equal(many.result.structuredContent.route.choice, "technical"); + assert.equal(many.result.structuredContent.urgency.choice, "high"); + assert.equal(requests.length, 3); + for (const received of requests.slice(1)) { + assert.deepEqual(received.body.messages[0].images, [image.toString("base64")]); + } }); test("serves a minimal-prefix choice using the configured llama.cpp endpoint", async (t) => { @@ -326,5 +343,17 @@ test("serves a minimal-prefix choice using the configured llama.cpp endpoint", a assert.ok(scoredTokenIds.includes("t".charCodeAt(0) + 1)); assert.ok(!scoredTokenIds.includes("A".charCodeAt(0) + 1)); assert.ok(requests.some((request) => request.body.model === "fixture-model")); + const many = await process_.rpc("tools/call", { + name: "chooseMany", + arguments: { + context: request.context, + questions: { + route: { question: request.question, choices: request.choices }, + followup: { question: "Who should investigate?", choices: request.choices }, + }, + }, + }); + assert.equal(many.result.structuredContent.route.choice, "billing"); + assert.equal(many.result.structuredContent.followup.choice, "billing"); for (const line of process_.lines) assert.doesNotThrow(() => JSON.parse(line)); }); diff --git a/packages/choosekit-mcp/tests/server.test.mjs b/packages/choosekit-mcp/tests/server.test.mjs index fcacb6d..1224483 100644 --- a/packages/choosekit-mcp/tests/server.test.mjs +++ b/packages/choosekit-mcp/tests/server.test.mjs @@ -86,6 +86,110 @@ async function callChoose(connection, arguments_) { return connection.rpc("tools/call", { name: "choose", arguments: arguments_ }); } +const manyRequest = { + context: request.context, + questions: { + route: { question: request.question, choices: request.choices }, + constructor: { question: "How urgent is this?", choices: { low: "Can wait", high: "Needs immediate attention" } }, + }, +}; + +test("chooseMany starts independent questions together and loads shared images once", async (t) => { + const calls = []; + const images = [{ mediaType: "image/png", base64: "aW1hZ2U=" }]; + let loads = 0; + let release; + const gate = new Promise((resolve) => { release = resolve; }); + const chooser = async (input) => { + calls.push(input); + if (calls.length === 2) release(); + await gate; + const keys = Object.keys(input.choices); + return { + choice: keys[0], + distribution: Object.fromEntries(keys.map((key, i) => [key, i === 0 ? 1 : 0])), + scores: Object.fromEntries(keys.map((key, i) => [key, i === 0 ? -0.1 : -Infinity])), + margin: 1, + entropy: 0, + boundaryTokens: 0, + usage: { promptTokens: input.question.length, cachedTokens: null, completionTokens: 1, requests: 1 }, + }; + }; + const connection = await connect(chooser, { + backend: "ollama", + imageLoader: async (paths) => { + loads++; + assert.deepEqual(paths, ["example.png"]); + return images; + }, + }); + t.after(() => connection.close()); + const response = await connection.rpc("tools/call", { + name: "chooseMany", arguments: { ...manyRequest, imagePaths: ["example.png"] }, + }); + assert.notEqual(response.result.isError, true); + const result = response.result.structuredContent; + assert.deepEqual(Object.keys(result), ["route", "constructor"]); + assert.equal(result.route.choice, "billing"); + assert.equal(result.constructor.choice, "low"); + assert.equal(result.route.scores.technical, null); + assert.equal(result.route.usage.promptTokens, request.question.length); + assert.equal(result.constructor.usage.promptTokens, manyRequest.questions.constructor.question.length); + assert.deepEqual(JSON.parse(response.result.content[0].text), result); + assert.equal(loads, 1); + for (const call of calls) { + assert.equal(call.context, manyRequest.context); + assert.equal(call.images, images); + assert.equal(call.signal, calls[0].signal); + } +}); + +test("chooseMany rejects invalid groups before calling the chooser", async (t) => { + let calls = 0; + const connection = await connect(async () => { calls++; }, { backend: "openrouter" }); + t.after(() => connection.close()); + const tooMany = Object.fromEntries(Array.from({ length: 21 }, (_, i) => [`choice${i}`, `Description ${i}`])); + for (const arguments_ of [ + { ...manyRequest, questions: {} }, + { ...manyRequest, model: "test-model" }, + { ...manyRequest, questions: { route: { ...manyRequest.questions.route, question: " " } } }, + { ...manyRequest, questions: { route: { ...manyRequest.questions.route, model: "test-model" } } }, + { ...manyRequest, questions: { route: { question: request.question, choices: tooMany } } }, + { ...manyRequest, questions: JSON.parse('{"__proto__":{"question":"Which?","choices":{"a":"First","b":"Second"}}}') }, + ]) { + const response = await connection.rpc("tools/call", { name: "chooseMany", arguments: arguments_ }); + assert.equal(response.result.isError, true); + assert.equal(calls, 0); + } +}); + +test("chooseMany returns a sanitized group error when a question fails", async (t) => { + const connection = await connect(async () => { throw new ScoringError("secret upstream details"); }); + t.after(() => connection.close()); + const response = await connection.rpc("tools/call", { name: "chooseMany", arguments: manyRequest }); + assert.equal(response.result.isError, true); + assert.equal(response.result.structuredContent, undefined); + assert.equal(response.result.content[0].text, "The model could not score the supplied choices."); +}); + +test("chooseMany passes MCP cancellation to every question", async (t) => { + const signals = []; + let started; + const hasStarted = new Promise((resolve) => { started = resolve; }); + const connection = await connect(({ signal }) => new Promise((resolve, reject) => { + signals.push(signal); + if (signals.length === 2) started(); + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + })); + t.after(() => connection.close()); + const call = await connection.startRpc("tools/call", { name: "chooseMany", arguments: manyRequest }); + await hasStarted; + await connection.notify("notifications/cancelled", { requestId: call.id, reason: "test" }); + assert.ok(signals.every((signal) => signal.aborted)); + connection.discard(call.id); + await assert.rejects(call.response, /cancelled by the test client/); +}); + test("exposes a strict, read-only llama.cpp choose tool", async (t) => { const connection = await connect(async () => assert.fail("chooser should not run")); t.after(() => connection.close());