From a1217e6e5b510eddaf42d8d61fa28f59c3004cb1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:45:15 +0000 Subject: [PATCH 1/2] Retire description/late-instruction It flagged instructions past character 280, a cut no client documents: the number came from one private client. The documented cuts (Claude Code 16,384 and 4,096, Amazon Q CLI 10,024) are description/buried's, as is any descriptionLimit a user sets. A config naming the old id is accepted and changes nothing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014kCPJZAgTv3Bna4BGLzw2o --- CHANGELOG.md | 8 ++++++ docs/PROTOCOLS.md | 10 +++---- docs/RULES.md | 1 - docs/SPEC.md | 1 - docs/snapshot.md | 3 +- src/rules/description.ts | 38 ------------------------- src/rules/index.ts | 11 ++++---- test/fixtures/golden/findings.json | 2 -- test/rules.test.mjs | 45 +++++------------------------- 9 files changed, 26 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 579ce95..c0448d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +### Changes + +- **`description/late-instruction` is retired.** It flagged instructions past + character 280, a cut that no client documents: the number came from one private + client. Documented cuts are 16,384 and 4,096 (Claude Code) and 10,024 (Amazon Q CLI), + and `description/buried` checks those, or your own `descriptionLimit`. A config that + still names the rule is accepted and changes nothing. + ## 0.28.1 ### Fixes diff --git a/docs/PROTOCOLS.md b/docs/PROTOCOLS.md index 82d6179..1395229 100644 --- a/docs/PROTOCOLS.md +++ b/docs/PROTOCOLS.md @@ -11,11 +11,11 @@ exactly the one in the row. | Protocol | Handshake | `tools/list` against its schema | Mid-session change, pinned to its step | A change only this session sees | `snapshot` rules that apply | |---|---|---|---|---|---| -| 2024-11-05 | `initialize` | ✓ | ✓ `session/mid-insert` (error), `list_changed` received | allowed: no finding | 41 of 47 | -| 2025-03-26 | `initialize` | ✓ | ✓ `session/mid-insert` (error), `list_changed` received | allowed: no finding | 41 of 47 | -| 2025-06-18 | `initialize` | ✓ | ✓ `session/mid-insert` (error), `list_changed` received | allowed: no finding | 41 of 47 | -| 2025-11-25 | `initialize` | ✓ | ✓ `session/mid-insert` (error), `list_changed` received | allowed: no finding | 41 of 47 | -| 2026-07-28 | `server/discover` | ✓ | ✓ `session/mid-insert` (error), `list_changed` received | `session/side-effect` (warn) | 47 of 47 | +| 2024-11-05 | `initialize` | ✓ | ✓ `session/mid-insert` (error), `list_changed` received | allowed: no finding | 40 of 46 | +| 2025-03-26 | `initialize` | ✓ | ✓ `session/mid-insert` (error), `list_changed` received | allowed: no finding | 40 of 46 | +| 2025-06-18 | `initialize` | ✓ | ✓ `session/mid-insert` (error), `list_changed` received | allowed: no finding | 40 of 46 | +| 2025-11-25 | `initialize` | ✓ | ✓ `session/mid-insert` (error), `list_changed` received | allowed: no finding | 40 of 46 | +| 2026-07-28 | `server/discover` | ✓ | ✓ `session/mid-insert` (error), `list_changed` received | `session/side-effect` (warn) | 46 of 46 | - **Handshake**: the request toolmenu connected with, and the server answering that version. - **`tools/list` against its schema**: `spec/schema` passes a valid result and reports an diff --git a/docs/RULES.md b/docs/RULES.md index 5fd2a96..310c0fa 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -22,7 +22,6 @@ means for semver: breaking → major, minor → minor, notice → no bump. | `catalog/skipped` | info | `--catalog`: tools that look like command routers but weren't called, because no listing mode needs no command. toolmenu never sends a command to list one | | `description/buried` | warn | A sentence that reads as an instruction to the agent sits past the point where the client cuts descriptions, so the model never sees it. Read from the words, so marked unsure | | `description/cut` | info | Descriptions longer than the client sends, with where the cut falls. Nothing past it reads as an instruction, so it's a check, not a bug | -| `description/late-instruction` | info | Only with the default cut: an instruction past 280 characters, where one real client cut, but inside the default cut. Split out of `description/cut` in 0.13; a setting for `description/cut` still applies to it unless it's set itself (`RULE_ALIASES`) | | `ids/authored` | info | An input that looks like an opaque ID where no tool appears to return that kind of ID, so the agent would have to invent it. Heuristic, and worded as one. Stronger when `outputSchema` is present | | `ids/handle-lifetime` | info | 2026-07-28. A tool that creates a handle (`create`/`start`/`open`/`new`/`begin`/`acquire` + session, basket, cart, checkout, handle, lease or lock) that another tool takes back (`session_id`, `sessionToken`, `session`), whose description and output schema don't say how long it lasts. The spec asks for retention in the creation tool's description. Unsure | | `lists/resource-read-schema` | error | `snapshot --read-resources`. Each `resources/read` result, as it came off the wire, against `ReadResourceResult` in the official schema of the negotiated protocol version: an item needs `uri` and `text` or `blob`; 2026-07-28 also requires `resultType`, `ttlMs` and `cacheScope`, earlier versions don't. One finding for the server, with the distinct schema errors, however many resources share them. A second kind: a result that matches the schema and the official SDK client still refused (seen with a `resultType` the schema takes as any string), in the SDK's words. An `InputRequiredResult` is checked against its own schema where the version has one. A version with no pinned schema isn't checked | diff --git a/docs/SPEC.md b/docs/SPEC.md index b0061f6..8f73f6a 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -331,7 +331,6 @@ Keyword matching only in v1: deterministic, free and reproducible in CI. |---|---|---| | `description/buried` | warn | A sentence that reads as an instruction to the agent sits past the point where the client cuts descriptions, so the model never sees it. Read from the words, so marked unsure | | `description/cut` | info | Descriptions longer than the client sends, with where the cut falls. Nothing past it reads as an instruction, so it's a check, not a bug | -| `description/late-instruction` | info | Only with the default cut: an instruction past 280 characters, where one real client cut, but inside the default cut. Split out of `description/cut` in 0.13; a setting for `description/cut` still applies to it unless it's set itself (`RULE_ALIASES`) | **Spec** (not observed failures: protocol requirements, kept minimal) diff --git a/docs/snapshot.md b/docs/snapshot.md index 8f97d73..ad5c448 100644 --- a/docs/snapshot.md +++ b/docs/snapshot.md @@ -21,7 +21,6 @@ server, or from the spec itself. If a rule can't point to one, it doesn't ship. | `naming/route` | error | A `routes.yml` expectation broke: a keyword now matches the wrong tool at least as well as the right one | | `description/buried` | warn | Instructions to the agent ("use X instead", "don't retry", "never guess one") past the point where your client cuts descriptions: in Claude Code 16,384 characters for a tool loaded through tool search (its default) and 4,096 for one that loads up front, or your `descriptionLimit`. From a real failure: a client cut at 280 characters, and the line that decided routing was at 1,222 | | `description/cut` | info | Descriptions longer than the client sends, with nothing that reads as an instruction past the cut, as one summary that shows where each one is cut. The model gets a prefix that can read as complete | -| `description/late-instruction` | info | With the default cut: instructions after character 280, which a client that cuts descriptions short would hide, as one summary. Set `descriptionLimit` and `description/buried` checks your real cut instead. (Before 0.13 this was part of `description/cut`; that id's setting still applies.) | | `naming/vague-id` | warn | Parameters called just `id` that don't say *which* thing, as one summary with a suggested name | | `ids/authored` | info | An ID the agent must supply that no tool appears to return, so the agent may invent it, one finding per kind of ID (heuristic: IDs from a URL or another tool's text output look the same) | | `ids/handle-lifetime` | info | 2026-07-28: a tool that creates a handle another tool takes back (`start_session` → `session_id`) doesn't say how long it lasts (heuristic) | @@ -163,7 +162,7 @@ Claude Code with tool search off) or `"amazon-q"` (10,024, reported in [makenotion/notion-mcp-server#145](https://github.com/makenotion/notion-mcp-server/issues/145)). Clients that send descriptions in full don't need a setting beyond that. If your own -client cuts shorter, as mine did at 280, set its number. `fullDescriptions` lists the +client cuts shorter, set its number. `fullDescriptions` lists the tools it sends uncut, so the description rules skip them. ## Every server you use: `scan` diff --git a/src/rules/description.ts b/src/rules/description.ts index ca4f534..da4c757 100644 --- a/src/rules/description.ts +++ b/src/rules/description.ts @@ -152,9 +152,6 @@ export const buried: Rule = { }, }; -/** A short cut to check against when the client's own cut isn't set: one real client cut at 280. */ -const SHORT_CUT = 280; - /** “…the last words before ✂ the first words after…”, to find the cut in the text. */ function aroundTheCut(text: string, at: number): string { const before = text.slice(Math.max(0, at - 30), at).replace(/\s+/g, ' '); @@ -197,38 +194,3 @@ export const cut: Rule = { }, }; -export const lateInstruction: Rule = { - id: 'description/late-instruction', - scope: 'agent', - severity: 'info', - lesson: 'truncated description', - summary: 'Instructions to the agent that a client cutting descriptions short would hide', - run(ctx) { - // Only with the default cut: once descriptionLimit is set, description/buried checks the real one. - if (ctx.descriptionLimit !== undefined) return []; - const { limit, who } = resolveLimit(undefined); - if (limit <= SHORT_CUT) return []; - const all = ctx.menu.tools.map((t) => t.name); - const late: { tool: MenuTool; at: number; straddles: boolean; sentence: string; more: number }[] = []; - for (const tool of kept(ctx.menu.tools, ctx)) { - const text = tool.description ?? ''; - if (text.length <= SHORT_CUT || sentInFull(tool, ctx.fullDescriptions)) continue; - // Past the default cut is description/buried's. - const hidden = pastTheCut(text, all.filter((n) => n !== tool.name), SHORT_CUT).filter((i) => i.at < limit); - if (hidden.length) late.push({ tool, at: hidden[0].at, straddles: hidden[0].straddles, sentence: sentenceAround(text, hidden[0].at), more: hidden.length - 1 }); - } - if (late.length === 0) return []; - const one = late.length === 1; - const client = who; - return [ - { - ...(one ? { tool: late[0].tool.name } : {}), - confidence: 'unsure', - message: `${one ? `The ${late[0].tool.name} description gives` : `${late.length} descriptions give`} the agent instructions ${late.every((l) => l.straddles) ? `that run past character ${SHORT_CUT}` : late.some((l) => l.straddles) ? `after character ${SHORT_CUT} or running past it` : `after character ${SHORT_CUT}`}, where a client that cuts descriptions short never shows them (one real client cut at ${SHORT_CUT}). ${client} sends ${num(limit)} characters, so there they arrive.`, - // One that starts before the cut and runs past it is cut in the middle: say so, or "char 264" reads as before the cut. - detail: late.map((l) => `${l.tool.name}: char ${num(l.at)}${l.straddles ? ` (runs past ${SHORT_CUT})` : ''}: “${clip(l.sentence, 120)}”${l.more ? ` (and ${l.more} more after it)` : ''}`), - fix: `If your client cuts descriptions, set descriptionLimit to its cut and toolmenu checks against that. Otherwise, moving ${one ? 'the instruction' : 'each instruction'} into the first sentences is cheap insurance.`, - }, - ]; - }, -}; diff --git a/src/rules/index.ts b/src/rules/index.ts index 4fd47ba..d79fbff 100644 --- a/src/rules/index.ts +++ b/src/rules/index.ts @@ -1,6 +1,6 @@ import type { Finding, Severity } from '../types.js'; import { SEVERITY_RANK } from '../types.js'; -import { buried, cut, lateInstruction } from './description.js'; +import { buried, cut } from './description.js'; import { connectionVariance, hostSpecific, nondeterministic, processVariance } from './determinism.js'; import { authoredIds } from './ids.js'; import { clientFeatures, clientLimit, clientName } from './clients.js'; @@ -43,7 +43,6 @@ export const MENU_RULES: Rule[] = [ handleLifetime, buried, cut, - lateInstruction, noDryRun, unannotated, readOnlyMismatch, @@ -75,11 +74,11 @@ export const MENU_RULES: Rule[] = [ ]; /** - * Rules split out of an older one. A setting for the old id still applies to the - * new one unless the new id is set itself: before 0.13, description/cut held - * the late-instruction hint too, and `"description/cut": "off"` turned both off. + * Rules split out of an older one: a setting for the old id still applies to the new + * one unless the new id is set itself. None now: description/late-instruction, split + * out of description/cut in 0.13, was retired after 0.28.1. */ -export const RULE_ALIASES: Record = { 'description/late-instruction': 'description/cut' }; +export const RULE_ALIASES: Record = {}; export interface RuleSettings { /** Per-rule severity override, or 'off'. */ diff --git a/test/fixtures/golden/findings.json b/test/fixtures/golden/findings.json index 6365ed3..b3d5d6e 100644 --- a/test/fixtures/golden/findings.json +++ b/test/fixtures/golden/findings.json @@ -1,7 +1,6 @@ { "snapshot": { "firecrawl-mcp-3.26.0.json": [ - "info description/late-instruction", "info search/findable firecrawl_scrape", "info write/no-dry-run", "warn naming/vague-id" @@ -34,7 +33,6 @@ "info write/no-dry-run" ], "supabase__mcp-server-supabase-0.13.0.json": [ - "info description/late-instruction get_publishable_keys", "info naming/shared-word", "info write/no-dry-run", "warn naming/vague-id" diff --git a/test/rules.test.mjs b/test/rules.test.mjs index 0c85dac..1068e30 100644 --- a/test/rules.test.mjs +++ b/test/rules.test.mjs @@ -426,8 +426,7 @@ test('description limits: Claude Code by default, presets, or your number', () = // with tool search off, every tool is cut at 4,096 assert.match(run([tool('get_record', ['record_id'], { description: text(5000) })], { descriptionLimit: 'claude-code-upfront' }).find((f) => f.rule === 'description/buried').message, /past character 4,096/); // before 16,384 but past 280: fine unless you say your client cuts at 280 - // (one summary for the length, one hint for the instruction a shorter cut would hide) - assert.deepEqual(run([tool('get_record', ['record_id'], { description: text(500) })]).map((f) => f.rule), ['description/cut', 'description/late-instruction']); + assert.deepEqual(run([tool('get_record', ['record_id'], { description: text(500) })]).map((f) => f.rule), ['description/cut']); assert.match(run([tool('get_record', ['record_id'], { description: text(500) })], { descriptionLimit: 280 })[0].message, /past character 280, where your client cuts the description/); // Amazon Q sends up to 10,024 assert.match(run([tool('get_record', ['record_id'], { description: text(10_100) })], { descriptionLimit: 'amazon-q' }).find((f) => f.rule === 'description/buried').message, /past character 10,024, where Amazon Q CLI cuts the description/); @@ -471,24 +470,14 @@ test('naming/route: a tool missing from a server that changes its menu is info, assert.equal(fixed[0].severity, 'error'); }); -test('description/late-instruction: with the default cut, instructions a shorter client cut would hide get one hint', () => { +test('description/late-instruction is retired: no client documents a cut at 280, so an instruction there says nothing', () => { const pad = 'Returns the record with every field it has. '.repeat(10); // 450 characters, under Claude Code's cut const tools = [tool('get_record', ['record_id'], { description: `${pad}Never guess a record ID.` }), tool('get_other', [], { description: 'Short.' })]; - const hint = lint(tools).filter((f) => f.rule === 'description/late-instruction'); - assert.equal(hint.length, 1); - assert.equal(hint[0].tool, 'get_record'); - assert.equal(hint[0].confidence, 'unsure'); - assert.match(hint[0].message, /^The get_record description gives the agent instructions after character 280.*Claude Code sends 16,384 characters/); - assert.deepEqual(hint[0].detail, ['get_record: char 440: “Never guess a record ID.”']); - assert.match(hint[0].fix, /set descriptionLimit/); - assert.deepEqual(lint(tools).filter((f) => f.rule === 'description/cut'), [], 'no longer a description/cut finding'); - // once the cut is set, the hint goes and buried decides + assert.deepEqual(lint(tools).filter((f) => f.rule.startsWith('description/')), []); + // a config that still names the old id changes nothing and isn't an error + assert.deepEqual(lint(tools, {}, { rules: { 'description/late-instruction': 'warn' } }).filter((f) => f.rule.startsWith('description/')), []); + // a client that does cut at 280, said with descriptionLimit, gets description/buried assert.deepEqual(lint(tools, { descriptionLimit: 280 }).filter((f) => f.rule.startsWith('description/')).map((f) => f.rule), ['description/buried']); - assert.deepEqual(lint(tools, { descriptionLimit: 'claude-code' }).filter((f) => f.rule === 'description/late-instruction'), []); - // the old id still turns it off, unless the new one is set itself - assert.deepEqual(lint(tools, {}, { rules: { 'description/cut': 'off' } }).filter((f) => f.rule.startsWith('description/')), []); - assert.equal(lint(tools, {}, { rules: { 'description/cut': 'warn' } }).find((f) => f.rule === 'description/late-instruction').severity, 'warn'); - assert.equal(lint(tools, {}, { rules: { 'description/cut': 'off', 'description/late-instruction': 'info' } }).filter((f) => f.rule === 'description/late-instruction').length, 1); }); test('ids/authored: abbreviations, lone qualifiers, URL provenance, one finding per kind (FINDINGS F12)', () => { @@ -593,7 +582,7 @@ test('certainty: guesses from words are marked unsure, observed facts are not', const found = lint(tools, { secondList: second, listMeta: { ttlMs: 0 }, protocolVersion: '2026-07-28', era: 'modern' }); const unsure = [...new Set(found.filter((f) => f.confidence === 'unsure').map((f) => f.rule))].sort(); const sure = [...new Set(found.filter((f) => !f.confidence).map((f) => f.rule))].sort(); - assert.deepEqual(unsure, ['description/buried', 'description/late-instruction', 'ids/authored', 'naming/shared-word', 'naming/vague-id', 'write/no-dry-run', 'write/unannotated'].filter((r) => unsure.includes(r))); + assert.deepEqual(unsure, ['description/buried', 'ids/authored', 'naming/shared-word', 'naming/vague-id', 'write/no-dry-run', 'write/unannotated'].filter((r) => unsure.includes(r))); for (const rule of ['naming/vague-id', 'naming/shared-word', 'ids/authored', 'write/unannotated']) assert.ok(unsure.includes(rule), rule); for (const rule of ['menu/nondeterministic', 'menu/duplicate-name', 'spec/cache-hints']) assert.ok(sure.includes(rule), rule); // and every finding gives a next step @@ -660,16 +649,6 @@ test('menu/process-variance: a second cause behind the same tool is reported, no assert.equal(found.find((x) => x.rule === 'menu/nondeterministic').fix.startsWith('Build jira_search.end_date.default'), true); }); -test('description/late-instruction: an instruction that starts before 280 and runs past it is said to run past it', () => { - const pad = 'Returns the record with every field it has. '.repeat(6); // 264 characters - // "Never guess" starts at 275 and ends at 286: across the cut. - const tools = [tool('get_record', ['record_id'], { description: `${pad}Note this: Never guess a record ID.` })]; - const hint = lint(tools).find((f) => f.rule === 'description/late-instruction'); - assert.match(hint.message, /^The get_record description gives the agent instructions that run past character 280/); - assert.equal(hint.detail.length, 1); - assert.match(hint.detail[0], /^get_record: char 275 \(runs past 280\): “.*Never guess a record ID\.”$/); -}); - const unlock = () => tool('unlock_toolset', [], { description: 'Unlock more tools.', @@ -752,16 +731,6 @@ test('naming/shared-word: a word in every tool name is the server\'s subject, no assert.deepEqual(words([tool('ask_wiki_question'), tool('read_wiki_contents'), tool('search_repos')]), ['wiki']); }); -test('description/late-instruction: a mix of instructions past and across the cut is said as both', () => { - const pad = 'Returns the record with every field it has. '.repeat(6); // 264 characters - const tools = [ - tool('get_record', ['record_id'], { description: `${pad}Note this: Never guess a record ID.` }), // across: starts at 275 - tool('get_other', ['other_id'], { description: `${'Returns the other thing with every field. '.repeat(10)}Never guess an other ID.` }), // after: 420 - ]; - const hint = lint(tools).find((f) => f.rule === 'description/late-instruction'); - assert.match(hint.message, /^2 descriptions give the agent instructions after character 280 or running past it/); -}); - test('naming/shared-word: a word that opens several tool names is a namespace, not a shared noun', () => { const words = (tools) => lint(tools).find((f) => f.rule === 'naming/shared-word')?.words.map((w) => w.noun) ?? []; assert.deepEqual(words([tool('content_get_page'), tool('content_update_page'), tool('export_content_report'), tool('list_team_audits'), tool('get_audit_trail')]), ['audit']); From 6db120da12eba17049eaafd1b8709cd4d7be8679 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 11:31:04 +0000 Subject: [PATCH 2/2] Retired rules: explain and config say so; SPEC keeps its table - RETIRED_RULES replaces RULE_ALIASES, which had no entries left. toolmenu explain on a retired id says it was retired and why; a config that names one loads, with a line saying the setting changes nothing. - docs/SPEC.md is frozen: its row stays, with a dated note. - CHANGELOG: Amazon Q's 10,024 is reported, not documented. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014kCPJZAgTv3Bna4BGLzw2o --- CHANGELOG.md | 7 ++++--- docs/SPEC.md | 5 +++++ src/cli.ts | 6 ++++++ src/config.ts | 2 ++ src/rules/index.ts | 12 +++++++----- test/cli.test.mjs | 9 +++++++++ test/explain.test.mjs | 10 ++++++++++ 7 files changed, 43 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0448d8..ec42e7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,10 @@ - **`description/late-instruction` is retired.** It flagged instructions past character 280, a cut that no client documents: the number came from one private - client. Documented cuts are 16,384 and 4,096 (Claude Code) and 10,024 (Amazon Q CLI), - and `description/buried` checks those, or your own `descriptionLimit`. A config that - still names the rule is accepted and changes nothing. + client. Claude Code documents 16,384 and 4,096; `description/buried` checks those, + the `amazon-q` preset's 10,024 (reported, not documented), or your own + `descriptionLimit`. A config that still names the rule loads, with a line saying it + changes nothing; `toolmenu explain description/late-instruction` says why it went. ## 0.28.1 diff --git a/docs/SPEC.md b/docs/SPEC.md index 8f73f6a..d6dbbaa 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -331,9 +331,14 @@ Keyword matching only in v1: deterministic, free and reproducible in CI. |---|---|---| | `description/buried` | warn | A sentence that reads as an instruction to the agent sits past the point where the client cuts descriptions, so the model never sees it. Read from the words, so marked unsure | | `description/cut` | info | Descriptions longer than the client sends, with where the cut falls. Nothing past it reads as an instruction, so it's a check, not a bug | +| `description/late-instruction` | info | Only with the default cut: an instruction past 280 characters, where one real client cut, but inside the default cut. Split out of `description/cut` in 0.13; a setting for `description/cut` still applies to it unless it's set itself (`RULE_ALIASES`) | **Spec** (not observed failures: protocol requirements, kept minimal) +> **Note, 2026-11-18 (after the freeze).** `description/late-instruction` is retired: +> 280 was one private client's cut, and no client documents it. `description/buried` +> checks the documented cuts, or the `descriptionLimit` a user sets. + | id | sev | check | |---|---|---| | `spec/x-mcp-header` | error / warn | `since: 2026-07-28`. An `x-mcp-header` annotation the spec doesn't allow (SEP-2243): empty; not an RFC 9110 token; the same header twice ignoring case; on a parameter that isn't `string`, `integer` or `boolean`, or has no single `type`; anywhere but a parameter reached through `properties` alone. Streamable HTTP clients **MUST** leave such a tool out of `tools/list`, and the official SDK client does, with only a console warning. Error on an HTTP server, warn on stdio (clients there MAY ignore the annotation). A `number` parameter counts too: the SDK client drops those tools since 2.4. Every broken annotation in a tool is listed (the SDK stops at the first). Checked against the SDK client's behaviour in the tests | diff --git a/src/cli.ts b/src/cli.ts index 96e41fa..0c8c5f8 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -38,6 +38,7 @@ import { CLI_OPTIONS } from './options.js'; import { editDistance } from './words.js'; import { formatScan, ListError, loadList, scan, scanExit, type AllowEnv } from './scan.js'; import { explain, findRule, formatExplained, listRules, ruleDocs } from './rule-docs.js'; +import { RETIRED_RULES } from './rules/index.js'; import { commandHelp, commandOptions, isCommand, overview, unknownCommand, type Command } from './help.js'; class UsageError extends Error {} @@ -593,6 +594,11 @@ function explainCommand(rest: string[], command: string[] | undefined, format: F process.stdout.write(format === 'json' ? `${JSON.stringify(Object.keys(ruleDocs()))}\n` : listRules()); return 0; } + const retired = rest[0].trim().toLowerCase(); + if (Object.hasOwn(RETIRED_RULES, retired)) { + process.stdout.write(format === 'json' ? `${JSON.stringify({ id: retired, retired: RETIRED_RULES[retired] })}\n` : `${retired}: ${RETIRED_RULES[retired]}\n`); + return 0; + } const found = findRule(rest[0]); if ('near' in found) { const hint = found.near.length ? ` Did you mean ${found.near.join(', ')}?` : ''; diff --git a/src/config.ts b/src/config.ts index 85bd76f..dd2d177 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,6 +3,7 @@ import { readFile } from 'node:fs/promises'; import { CLIENTS } from './clients.js'; import { LIST_WORDS } from './lists.js'; import { CLIENT_LIMITS } from './rules/description.js'; +import { RETIRED_RULES } from './rules/index.js'; import type { Severity } from './types.js'; import { jsonErrorAt } from './menu.js'; @@ -70,6 +71,7 @@ export async function loadConfig(path?: string): Promise { if (!['error', 'warn', 'info', 'off'].includes(value)) { throw new Error(`${file}: rules["${id}"] must be error, warn, info or off`); } + if (Object.hasOwn(RETIRED_RULES, id)) process.stderr.write(`toolmenu: ${file}: rules["${id}"] changes nothing: ${RETIRED_RULES[id]}\n`); } if (config.protocol !== undefined && !['auto', 'legacy', 'modern'].includes(config.protocol)) { throw new Error(`${file}: protocol must be auto, legacy or modern`); diff --git a/src/rules/index.ts b/src/rules/index.ts index d79fbff..a71d550 100644 --- a/src/rules/index.ts +++ b/src/rules/index.ts @@ -74,11 +74,13 @@ export const MENU_RULES: Rule[] = [ ]; /** - * Rules split out of an older one: a setting for the old id still applies to the new - * one unless the new id is set itself. None now: description/late-instruction, split - * out of description/cut in 0.13, was retired after 0.28.1. + * Rules that no longer exist, and why. A config may still name one: it changes nothing, + * and loading the config says so. `explain` says the same. */ -export const RULE_ALIASES: Record = {}; +export const RETIRED_RULES: Record = { + 'description/late-instruction': + "retired after 0.28.1: it checked a cut at 280 characters, which no client documents. If your client cuts descriptions, set descriptionLimit to its cut and description/buried checks against it.", +}; export interface RuleSettings { /** Per-rule severity override, or 'off'. */ @@ -94,7 +96,7 @@ export function runRules(rules: Rule[], ctx: RuleContext, settings: RuleSettings // The menu as an MCP Apps host shows it to the agent: app-only tools left out. const shown = agentMenu(ctx.menu).shown; for (const rule of rules) { - const configured = settings.rules?.[rule.id] ?? (RULE_ALIASES[rule.id] ? settings.rules?.[RULE_ALIASES[rule.id]] : undefined); + const configured = settings.rules?.[rule.id]; if (configured === 'off') continue; if (rule.since && (!ctx.protocolVersion || ctx.protocolVersion < rule.since)) continue; for (const f of rule.run({ ...ctx, ...(rule.scope === 'agent' ? { menu: shown } : {}), isIgnored })) { diff --git a/test/cli.test.mjs b/test/cli.test.mjs index cec127d..7dde166 100644 --- a/test/cli.test.mjs +++ b/test/cli.test.mjs @@ -131,6 +131,15 @@ test('config file changes severities', async () => { assert.match(r.stdout, /ERROR naming\/vague-id/); }); +test('a config that names a retired rule loads, and says the setting changes nothing', async () => { + const cwd = tempDir(); + writeFileSync(join(cwd, 'toolmenu.config.json'), JSON.stringify({ rules: { 'description/late-instruction': 'error' } })); + const r = await run(['snapshot', '--no-write', ...raw('smells')], { cwd }); + assert.notEqual(r.code, 2, r.stderr); + assert.match(r.stderr, /rules\["description\/late-instruction"\] changes nothing: retired after 0\.28\.1/); + assert.doesNotMatch(r.stdout, /late-instruction/); +}); + test('github format prints workflow annotations', async () => { const r = await run(['snapshot', '--no-write', '--format', 'github', ...raw('shuffle')]); assert.match(r.stdout, /^::error title=toolmenu menu\/nondeterministic::/m); diff --git a/test/explain.test.mjs b/test/explain.test.mjs index ef36cc1..64f7cf6 100644 --- a/test/explain.test.mjs +++ b/test/explain.test.mjs @@ -67,3 +67,13 @@ test('explain: an unknown rule names the closest; no rule lists them all', async assert.equal(server.code, 2); assert.match(server.stderr, /not a server/); }); + +test('explain: a retired rule says it was retired and why, in text and JSON', async () => { + const text = await run(['explain', 'description/late-instruction']); + assert.equal(text.code, 0, text.stderr); + assert.match(text.stdout, /^description\/late-instruction: retired after 0\.28\.1: .*no client documents.*descriptionLimit/); + const json = await run(['explain', 'description/late-instruction', '--json']); + assert.equal(json.code, 0, json.stderr); + assert.equal(JSON.parse(json.stdout).id, 'description/late-instruction'); + assert.match(JSON.parse(json.stdout).retired, /^retired after 0\.28\.1/); +});