From 9d2ba0e9baeb333ccc69da1dca65ea391aeb6e7c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 10:23:22 +0000 Subject: [PATCH 1/3] 0.28.0: version, changelog, Action pins Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014kCPJXf7SykHNoEUW3LYNt --- CHANGELOG.md | 2 +- README.md | 2 +- docs/ci.md | 2 +- docs/github-action.md | 4 ++-- package-lock.json | 4 ++-- package.json | 2 +- server.json | 4 ++-- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e07e56..f4b5f04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.28.0 ### Fixes diff --git a/README.md b/README.md index 2cc1c7c..9b9833c 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ jobs: - uses: actions/setup-node@v7 with: { node-version: 22 } - run: npm ci && npm run build - - uses: niksa90/toolmenu@v0.27.1 + - uses: niksa90/toolmenu@v0.28.0 with: command: node dist/server.js baseline: menu.json # your committed snapshot diff --git a/docs/ci.md b/docs/ci.md index 015b73f..405960e 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -24,7 +24,7 @@ permissions: security-events: write # to upload SARIF steps: # … checkout, build, as above - - uses: niksa90/toolmenu@v0.27.1 + - uses: niksa90/toolmenu@v0.28.0 with: command: node dist/server.js sarif: toolmenu-sarif diff --git a/docs/github-action.md b/docs/github-action.md index 2678bd3..b777d2b 100644 --- a/docs/github-action.md +++ b/docs/github-action.md @@ -204,7 +204,7 @@ that adds findings at or above `fail-on` fails even when nothing breaks. Snapsho session findings count either way. ```yaml - - uses: niksa90/toolmenu@v0.27.1 + - uses: niksa90/toolmenu@v0.28.0 with: command: node dist/server.js gate: review @@ -247,7 +247,7 @@ version in a comment, and let Dependabot's `github-actions` updates keep the pin current: ```yaml - - uses: niksa90/toolmenu@ # v0.27.1 + - uses: niksa90/toolmenu@ # v0.28.0 ``` `toolmenu init` writes the actions it sets up (checkout, setup-node, …) pinned this diff --git a/package-lock.json b/package-lock.json index aa892b2..8a97368 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "toolmenu", - "version": "0.27.1", + "version": "0.28.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "toolmenu", - "version": "0.27.1", + "version": "0.28.0", "license": "MIT", "dependencies": { "@modelcontextprotocol/client": "^2.4.0", diff --git a/package.json b/package.json index b6d3951..fb27cd8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "toolmenu", - "version": "0.27.1", + "version": "0.28.0", "mcpName": "io.github.niksa90/toolmenu", "description": "Test your MCP server the way agents and clients meet it: the menu, the calls, the results, the sign-in, and every release after. No LLM involved.", "author": "Nikša Perović (https://niksa.me)", diff --git a/server.json b/server.json index d4cedf6..c5cf0dc 100644 --- a/server.json +++ b/server.json @@ -14,13 +14,13 @@ "mimeType": "image/svg+xml" } ], - "version": "0.27.1", + "version": "0.28.0", "packages": [ { "registryType": "npm", "registryBaseUrl": "https://registry.npmjs.org", "identifier": "toolmenu", - "version": "0.27.1", + "version": "0.28.0", "runtimeHint": "npx", "transport": { "type": "stdio" From 6d540f670929812e08a42e71af1a98499ef5d79f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 10:35:47 +0000 Subject: [PATCH 2/3] ci: the PR comment is toolmenu mcp's own menu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Action's PR comment came from its dogfood run on test/fixtures/sdk-server.mjs, three tools, so every PR here reported "3 tools" and never toolmenu's own server. The Action keeps one comment per PR, so the fixture run (still testing the Action) now posts none, and a second run on `node dist/cli.js mcp` against the new docs/toolmenu-menu.json posts it: 6 tools, the dry-run hint, and a diff for any PR that changes a tool's name, description or schema. test/mcp.test.mjs fails when docs/toolmenu-menu.json or the badge no longer match the server; AGENTS.md's command writes both. A version bump alone is no difference (diff reports nothing). Tests: npm test, 725 passing. action/run.sh on toolmenu mcp with that baseline: verdict unchanged, 6 → 6 tools. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014kCPJXf7SykHNoEUW3LYNt --- .github/workflows/ci.yml | 10 +- AGENTS.md | 7 +- CHANGELOG.md | 7 + docs/toolmenu-menu.json | 275 +++++++++++++++++++++++++++++++++++++++ test/mcp.test.mjs | 13 ++ 5 files changed, 309 insertions(+), 3 deletions(-) create mode 100644 docs/toolmenu-menu.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe86b27..94129c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,8 @@ jobs: - run: npm run build - run: node --test --test-reporter=spec test/mcp.test.mjs - # Dogfood: the Action from this checkout, on its own fixture server. + # Dogfood: the Action from this checkout, on its own fixture server, then on toolmenu + # mcp (the PR comment: one per PR, so only the second run posts it). action: runs-on: ubuntu-latest permissions: @@ -69,6 +70,13 @@ jobs: command: node test/fixtures/sdk-server.mjs baseline: examples/menu.json release: 'off' # package.json is toolmenu's version, not the fixture server's + comment: 'false' + - uses: ./ + with: + cli: node dist/cli.js + command: node dist/cli.js mcp + baseline: docs/toolmenu-menu.json + release: 'off' # the version is bumped at release, not in each PR # The Action on a job whose Node is older than 22: toolmenu gets its own Node 22, # and the server and the job's later steps keep Node 20. diff --git a/AGENTS.md b/AGENTS.md index d7ce12f..48429b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,9 @@ findings on every run. - `docs/snapshot.md`, `session.md`, `diff.md`, `clients.md`, `usage.md`, `ci.md`, `github-action.md`: the reference, one page per command or area. - `src/mcp.ts`: `toolmenu mcp`, the tools an agent calls. Its own menu must pass - toolmenu with no finding but the dry-run hint; `docs/toolmenu-badge.json` is that run. + toolmenu with no finding but the dry-run hint; `docs/toolmenu-menu.json` and + `docs/toolmenu-badge.json` are that run. CI runs the Action on it against that menu file, + and that run is the PR comment. - `plugins/toolmenu/`: the Claude Code plugin (skills, evals, the pinned release). - `.claude/agents/bench-finding-checker.md`: a read-only subagent that checks one bench finding against the run's raw output. @@ -111,7 +113,8 @@ A release is one commit, then a `v*` tag; the tag publishes to npm (`release.yml pins a published release, never one ahead of npm; a test checks. A change to `src/mcp.ts` that alters a tool's name, description or schema needs -`node dist/cli.js snapshot --no-write --badge docs/toolmenu-badge.json -- node dist/cli.js mcp`, +`node dist/cli.js snapshot --out docs/toolmenu-menu.json --badge docs/toolmenu-badge.json -- node dist/cli.js mcp` +(a test fails until it's run), and the skills in `plugins/toolmenu/skills/` reread: they name the tools. ## Pull request descriptions diff --git a/CHANGELOG.md b/CHANGELOG.md index f4b5f04..ac3dfaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,13 @@ request each. No content is kept or quoted. The reads aren't in the menu file. +### Repository + +- **The PR comment is toolmenu's own server.** CI runs the Action on `toolmenu mcp` + (6 tools) against the new `docs/toolmenu-menu.json`; the run on the three-tool fixture + still tests the Action, without a comment. A test fails when the committed menu file or + badge no longer match the server. + ## 0.27.1 ### New diff --git a/docs/toolmenu-menu.json b/docs/toolmenu-menu.json new file mode 100644 index 0000000..5153fb6 --- /dev/null +++ b/docs/toolmenu-menu.json @@ -0,0 +1,275 @@ +{ + "toolmenu": 1, + "server": { + "name": "toolmenu", + "version": "0.28.0", + "protocolVersion": "2026-07-28", + "era": "modern", + "extensions": {} + }, + "capturedAt": "2026-11-11T10:27:35.404Z", + "tools": [ + { + "name": "snapshot_menu", + "description": "Check an MCP server's tool list as a client reads it: start or connect to it, list its tools, and report findings (what clients reject, what breaks prompt caches, what agents misread) and the token cost. A command server is started by running command as given, so only pass a command you would run yourself. Writes the menu file only when save_as is given (the baseline for diff_menus): a new file inside cwd, or an existing menu file there.", + "inputSchema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "command": { + "description": "The command that starts a stdio server, one word per item: [\"node\", \"dist/server.js\"]. Give this or url.", + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "url": { + "description": "A Streamable HTTP endpoint: https://example.com/mcp. Give this or command.", + "type": "string" + }, + "headers": { + "description": "HTTP headers for a url server: {\"Authorization\": \"Bearer ...\"}.", + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + } + }, + "env": { + "description": "Environment variables for a command server. It gets a minimal environment plus these.", + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + } + }, + "cwd": { + "description": "Folder to run from: where the server starts, toolmenu.config.json is read and relative paths resolve. Default: where toolmenu mcp runs.", + "type": "string" + }, + "save_as": { + "description": "Write the menu here (menu.json for a baseline), relative to cwd and inside it: a new file, or a menu file to replace. Default: not written.", + "type": "string" + }, + "client": { + "description": "Also check the menu as this client loads it: what it cuts, finds and refuses.", + "type": "string", + "enum": [ + "claude-code", + "claude-api-search", + "openai", + "codex", + "vscode", + "copilot-cloud-agent", + "copilot-code-review", + "gemini-cli", + "gemini-api", + "antigravity" + ] + }, + "catalog": { + "description": "Also read the operations behind a search tool or command routers.", + "type": "boolean" + } + } + }, + "annotations": { + "title": "Snapshot a server menu", + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + }, + "tokens": 461 + }, + { + "name": "diff_menus", + "description": "Compare two menu files (from snapshot_menu save_as, or toolmenu snapshot): breaking changes for clients, the token change per conversation, and the version bump it needs. Reads files only.", + "inputSchema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "before": { + "type": "string", + "description": "The older menu file, e.g. the committed menu.json." + }, + "after": { + "type": "string", + "description": "The newer menu file." + }, + "release": { + "description": "The release versions to check the bump against: \"1.4.0..1.5.0\".", + "type": "string" + }, + "cwd": { + "description": "Folder to run from: where the server starts, toolmenu.config.json is read and relative paths resolve. Default: where toolmenu mcp runs.", + "type": "string" + } + }, + "required": [ + "before", + "after" + ] + }, + "annotations": { + "title": "Diff two menus", + "readOnlyHint": true, + "openWorldHint": false + }, + "tokens": 187 + }, + { + "name": "check_session", + "description": "Call an MCP server's tools as an agent would and check what comes back: its tools marked readOnlyHint whose arguments the schema can fill are called, each result is checked against the tool's outputSchema, and the menu is listed after each call, with each change reported with the call that caused it. Calls only read-only tools; never one marked openWorldHint unless open_world is set. A command server is started by running command as given, so only pass a command you would run yourself.", + "inputSchema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "command": { + "description": "The command that starts a stdio server, one word per item: [\"node\", \"dist/server.js\"]. Give this or url.", + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "url": { + "description": "A Streamable HTTP endpoint: https://example.com/mcp. Give this or command.", + "type": "string" + }, + "headers": { + "description": "HTTP headers for a url server: {\"Authorization\": \"Bearer ...\"}.", + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + } + }, + "env": { + "description": "Environment variables for a command server. It gets a minimal environment plus these.", + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + } + }, + "cwd": { + "description": "Folder to run from: where the server starts, toolmenu.config.json is read and relative paths resolve. Default: where toolmenu mcp runs.", + "type": "string" + }, + "max_calls": { + "description": "At most this many calls (default 20).", + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "open_world": { + "description": "Also call read-only tools marked openWorldHint (web search, fetch): they may cost API credits.", + "type": "boolean" + } + } + }, + "annotations": { + "title": "Run a session against a server", + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": false, + "openWorldHint": true + }, + "tokens": 394 + }, + { + "name": "review_baseline", + "description": "Say whether a new menu file should replace the baseline: one verdict (unchanged, safe, breaking-released, breaking, review), what breaks callers, and which findings the new menu adds, clears or makes worse. Use it before overwriting a committed menu.json. Reads the two files only and starts no server.", + "inputSchema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "baseline": { + "type": "string", + "description": "The menu file to replace, e.g. the committed menu.json." + }, + "candidate": { + "type": "string", + "description": "The new menu file, e.g. new-menu.json from snapshot_menu save_as." + }, + "release": { + "description": "The release versions, so breaking changes pass when the bump allows them: \"1.4.0..2.0.0\".", + "type": "string" + }, + "cwd": { + "description": "Folder to run from: where the server starts, toolmenu.config.json is read and relative paths resolve. Default: where toolmenu mcp runs.", + "type": "string" + } + }, + "required": [ + "baseline", + "candidate" + ] + }, + "annotations": { + "title": "Review a new menu against the baseline", + "readOnlyHint": true, + "openWorldHint": false + }, + "tokens": 231 + }, + { + "name": "check_auth", + "description": "Check an HTTP MCP server's OAuth discovery the way a client that follows the spec walks it, without logging in: the 401 and its WWW-Authenticate, the protected resource metadata, the authorization server's metadata, PKCE and registration. Use it when a server answers 401 or a client cannot sign in. Sends unauthenticated requests to the URL and to the metadata addresses it names; no credentials, no tokens.", + "inputSchema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "url": { + "type": "string", + "description": "The Streamable HTTP endpoint: https://mcp.example.com/mcp." + } + }, + "required": [ + "url" + ] + }, + "annotations": { + "title": "Check a server's OAuth discovery", + "readOnlyHint": true, + "openWorldHint": true + }, + "tokens": 149 + }, + { + "name": "explain_rule", + "description": "Explain a toolmenu rule by the name a finding gives (naming/vague-id): what it checks, its default level and how to change it. Without rule, lists every rule.", + "inputSchema": { + "type": "object", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "rule": { + "description": "The rule, as a finding names it: naming/vague-id, or just vague-id.", + "type": "string" + } + } + }, + "annotations": { + "title": "Explain a rule", + "readOnlyHint": true, + "openWorldHint": false + }, + "tokens": 101 + } + ], + "totalTokens": 1523, + "listMeta": { + "ttlMs": 3600000, + "cacheScope": "public" + } +} diff --git a/test/mcp.test.mjs b/test/mcp.test.mjs index 1aa8a11..9bfd132 100644 --- a/test/mcp.test.mjs +++ b/test/mcp.test.mjs @@ -33,6 +33,19 @@ test('its own menu: no findings on either protocol, but the dry-run hint for the } }); +test('the committed menu file and badge are this build\'s: CI diffs PRs against them', async () => { + // Out of date, a PR that changes a tool's name, description or schema gets a diff of + // someone else's change in its comment. Regenerate with the command in AGENTS.md. + const dir = tempDir(); + const r = await run(['snapshot', '--out', join(dir, 'menu.json'), '--badge', join(dir, 'badge.json'), '--', process.execPath, CLI, 'mcp']); + assert.equal(r.code, 0, r.stdout + r.stderr); + const d = await run(['diff', '--json', join(ROOT, 'docs/toolmenu-menu.json'), join(dir, 'menu.json')]); + const diff = JSON.parse(d.stdout); + assert.deepEqual(diff.findings, [], 'docs/toolmenu-menu.json differs from toolmenu mcp: rerun the snapshot in AGENTS.md'); + assert.deepEqual(Object.values(diff.classes).map((c) => c.changes), [0, 0, 0]); + assert.equal(JSON.parse(readFileSync(join(dir, 'badge.json'), 'utf8')).message, JSON.parse(readFileSync(join(ROOT, 'docs/toolmenu-badge.json'), 'utf8')).message); +}); + test('snapshot_menu checks a stdio server and writes the menu only when asked', async () => { const cwd = tempDir(); await withServer(async (call) => { From 4721182beeb96ba160080ae312d23014a07002be Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 11:13:40 +0000 Subject: [PATCH 3/3] 0.28.0: the changelog in full, the last v0.16.1 pins CHANGELOG 0.28.0 gains the weekly bench's --read-resources and the build-vendors.py kind fix, and loses a broken line. The three examples in docs/github-action.md that still pinned v0.16.1 pin v0.28.0. The lists/resource-unread row says what the code says: something the run lacks, not the machine. --- CHANGELOG.md | 13 ++++++++++--- docs/RULES.md | 2 +- docs/github-action.md | 6 +++--- src/resource-read.ts | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac3dfaa..91deb91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,15 +29,22 @@ `--max-reads`, the 120 s running out. Templates and `http(s)://` URIs aren't read; a `ui://` resource a tool names stays with `apps/resource`. `--max-reads ` reads n, spread over the sorted list, one - request each. No - content is kept or quoted. The reads aren't in the menu file. + request each. No content is kept or quoted, and the reads aren't in the menu file. + FINDINGS F21 has the bench run: 53 servers, 6 of them listing 13 resources a client + can't read. ### Repository -- **The PR comment is toolmenu's own server.** CI runs the Action on `toolmenu mcp` +- **The PR comment checks toolmenu's own server.** CI runs the Action on `toolmenu mcp` (6 tools) against the new `docs/toolmenu-menu.json`; the run on the three-tool fixture still tests the Action, without a comment. A test fails when the committed menu file or badge no longer match the server. +- **The weekly bench reads resources:** `--read-resources` on stdio servers and images, + `--max-reads 5` on hosted ones, and snapshot's outer timeout from 300 to 420 s. + `build-vendors.py` filed `lists/resource-unreadable` under `lists/resource-unread`'s + kind (it matched rule names as prefixes); an entry without a trailing `/` now matches + one rule whole. +- `docs/github-action.md`: three examples still pinned `v0.16.1`. ## 0.27.1 diff --git a/docs/RULES.md b/docs/RULES.md index 62d6fec..5fd2a96 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -26,7 +26,7 @@ means for semver: breaking → major, minor → minor, notice → no bump. | `ids/authored` | info | An input that looks like an opaque ID where no tool appears to return that kind of ID, so the agent would have to invent it. Heuristic, and worded as one. Stronger when `outputSchema` is present | | `ids/handle-lifetime` | info | 2026-07-28. A tool that creates a handle (`create`/`start`/`open`/`new`/`begin`/`acquire` + session, basket, cart, checkout, handle, lease or lock) that another tool takes back (`session_id`, `sessionToken`, `session`), whose description and output schema don't say how long it lasts. The spec asks for retention in the creation tool's description. Unsure | | `lists/resource-read-schema` | error | `snapshot --read-resources`. Each `resources/read` result, as it came off the wire, against `ReadResourceResult` in the official schema of the negotiated protocol version: an item needs `uri` and `text` or `blob`; 2026-07-28 also requires `resultType`, `ttlMs` and `cacheScope`, earlier versions don't. One finding for the server, with the distinct schema errors, however many resources share them. A second kind: a result that matches the schema and the official SDK client still refused (seen with a `resultType` the schema takes as any string), in the SDK's words. An `InputRequiredResult` is checked against its own schema where the version has one. A version with no pinned schema isn't checked | -| `lists/resource-unread` | info | `snapshot --read-resources`. The listed resources no finding covers, by reason and with counts, so a clean result isn't read as complete: the run's failures (401/403 and other credential errors, payment, the network, a client capability, something this machine lacks, a read that didn't answer within `--timeout`), a rate limit (the reads stop at the first one), an `InputRequiredResult` (2026-07-28: the server MAY ask for input first), the 120 s for all reads running out, 64 MB of content having arrived, `--max-reads`, a listing that declares a `size` over 16 MB, `http(s)://` URIs (the spec has the client fetch those itself), a resource gone from the second `resources/list`, and a list cut at the page limit. An error-only body whose words are a credential or network failure counts here too. A missing backend counts when the server names it: "Invalid kube-config file", "You must provide an endpoint", "No currently active project" | +| `lists/resource-unread` | info | `snapshot --read-resources`. The listed resources no finding covers, by reason and with counts, so a clean result isn't read as complete: the run's failures (401/403 and other credential errors, payment, the network, a client capability, something the run lacks (a cluster, a project, an app), a read that didn't answer within `--timeout`), a rate limit (the reads stop at the first one), an `InputRequiredResult` (2026-07-28: the server MAY ask for input first), the 120 s for all reads running out, 64 MB of content having arrived, `--max-reads`, a listing that declares a `size` over 16 MB, `http(s)://` URIs (the spec has the client fetch those itself), a resource gone from the second `resources/list`, and a list cut at the page limit. An error-only body whose words are a credential or network failure counts here too. A missing backend counts when the server names it: "Invalid kube-config file", "You must provide an endpoint", "No currently active project" | | `lists/resource-unreadable` | warn | `snapshot --lists resources --read-resources` (or `readResources` in the config) sends `resources/read` for each concrete resource `resources/list` names: once per URI, in URI order, one at a time. Resource templates aren't read, and a `ui://` resource a tool names is `apps/resource`'s. warn: the read answers with a JSON-RPC error that isn't the run's, one finding with each URI and the server's words (`-32002`, `-32602`, `ENOENT` and "not found" are the server's here: it listed the resource). No spec version says a listed resource MUST be readable, so this rests on what toolmenu saw and is never an error. warn, marked unsure: the read succeeds and every content item is text holding a JSON object with an `error` key and no key beyond `error`, `message`, `code`, `status`, `detail`, `details`, `uri` (`@dpantani/tdmcp` 0.13.2 answered `{"error": "Python class … not found."}` for five listed resources); a guess from the shape, and the body is never quoted. warn, marked unsure: two or more reads fail, all in the same words (the URI aside), none says the resource is missing, the words name no cause toolmenu knows, and nothing reads. That is a read handler that fails for everything, or one thing the run lacks that the server didn't name; the finding says both. info: an empty `contents` array, which 2026-07-28 calls ambiguous. Each resource gets one `resources/read` request: the SDK client's own retries after an `InputRequiredResult` are off for these reads. Quiet: empty text, several contents, contents with other URIs (the spec allows sub-resources), a returned `uri` or `mimeType` that differs from the listing | | `lists/schema` | error | Each list's raw result pages against the official schema of the protocol version (`ListPromptsResult`, `ListResourcesResult`, `ListResourceTemplatesResult`); 2026-07-28 requires `ttlMs` and `cacheScope`. A declared list that fails to answer is a warning; Method not found (-32601) is info, and a setup failure (credentials, this machine) is info that says it wasn't checked. A list asked for whose capability the server doesn't declare isn't read, and the snapshot says so in a note, not a finding | | `lists/variance` | warn | `snapshot --lists prompts,resources` (or `lists` in the config) reads `prompts/list`, `resources/list` and `resources/templates/list` (only those the server declares a capability for, up to 50 pages each, bypassing the SDK's list cache) twice on the main connection and once from a second process or connection (`--processes 1` turns that off). A difference by item (name, uri, uriTemplate), in content, key order or order is the finding | diff --git a/docs/github-action.md b/docs/github-action.md index b777d2b..02f86e2 100644 --- a/docs/github-action.md +++ b/docs/github-action.md @@ -92,7 +92,7 @@ jobs: - run: npm ci && npm run build - run: npm start & # the PR's version, in the background (it keeps running for later steps) - run: npx --yes wait-on --timeout 60000 tcp:localhost:3000 # fail after 60 s instead of hanging - - uses: niksa90/toolmenu@v0.16.1 + - uses: niksa90/toolmenu@v0.28.0 with: url: http://localhost:3000/mcp headers: "x-mcp-api-key: ${{ env.MCP_API_KEY }}" @@ -111,7 +111,7 @@ tools behind an unlock: a renamed, removed or no-longer-read-only one never reac the report. Let the scenario build the baseline instead: ```yaml - - uses: niksa90/toolmenu@v0.16.1 + - uses: niksa90/toolmenu@v0.28.0 with: command: node dist/server.js scenario: scenario.yml # unlocks each toolset, lists after each @@ -174,7 +174,7 @@ check even though nothing was checked. The PR comment and a notice say so, and t `skipped` output is `'true'`. To make a skip fail instead, gate on it: ```yaml - - uses: niksa90/toolmenu@v0.16.1 + - uses: niksa90/toolmenu@v0.28.0 id: toolmenu with: url: https://mcp.example.com/mcp # a remote server that needs a key diff --git a/src/resource-read.ts b/src/resource-read.ts index 6c2c4dd..f0a8d93 100644 --- a/src/resource-read.ts +++ b/src/resource-read.ts @@ -256,7 +256,7 @@ export function runs(text: string, code: number | undefined): UnreadReason | und if (SETUP_WORDS.test(text)) return 'environment'; const failure: FailureClass = classifyFailure(text, { code }); if (failure === 'auth' || failure === 'payment' || failure === 'network' || failure === 'client') return failure; - // Something this machine lacks (a browser, an app). A missing file is not that: for a + // Something this run lacks (a browser, an app, a cluster). A missing file is not that: for a // listed resource, ENOENT is the server naming a file it doesn't have. if (failure === 'environment' && !/\bENOENT\b/.test(text)) return 'environment'; return undefined;