From 5103a54d5e4df1a4ff0d50943f96726ae3040068 Mon Sep 17 00:00:00 2001 From: integ Date: Thu, 3 Sep 2026 15:06:53 -0500 Subject: [PATCH 1/2] fix(openai): stop long sessions on OpenAI models dying with "Prompt is too long" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a conversation gets long, Claude Code shrinks it by asking the model for a plain-text summary. It sends that request with every tool still attached and relies on the wording of the prompt alone to stop the model using them. OpenAI-family models ignore that and answer with a tool call instead — usually whichever tool the session has been leaning on, such as Bash. Claude Code refuses to run the call and allows only one turn, so the attempt returns no summary at all and is thrown away. Three throwaways in a row and it stops shrinking the conversation, which then grows until the session stops dead with "Prompt is too long". Inside a background agent or a `-p` run nothing ever resets that, so the run is finished. clodex already made these requests text-only, but only recognised one when the session happened to carry Claude Code's structured-output tool. Sessions without it — an ordinary interactive session, a subagent spawned without a schema — got no protection: 15 of 173 such requests in local traffic logs, including the session this was diagnosed from. clodex now recognises the request by its own wording, whatever tools the session carries, which also covers manual `/compact`. The wording has to open the message, not merely appear in it, so that a turn which quotes the instructions — a pasted prompt, an agent's report, a read of clodex's own source — keeps its tools. Tool definitions are still sent so the cached prompt prefix keeps matching, and a request that does not match is left exactly as it was. --- .claude/docs/translation.md | 15 ++ src/sdk-adapter.ts | 44 ++++-- tests/sdk-adapter.test.ts | 266 ++++++++++++++++++++++++++++++++++++ 3 files changed, 312 insertions(+), 13 deletions(-) diff --git a/.claude/docs/translation.md b/.claude/docs/translation.md index f27d2601..591d83f7 100644 --- a/.claude/docs/translation.md +++ b/.claude/docs/translation.md @@ -22,6 +22,21 @@ hand-rolled per-provider translation. Preserved hard-won behavior: tokenizes at ~1.5 chars/token — 200k+ tokens per screenshot, killing agents with "Prompt is too long" while the local bytes/4 estimate showed half the real count. `estimateAnthropicInputTokens` likewise counts each image block at a flat vision estimate. +- **A compaction turn is forced to plain text with `toolChoice: 'none'`.** Claude Code forks that + turn — automatic and manual `/compact` alike — with the forking session's full tool list and + relies only on the prompt to stop the model calling them, while denying tool *execution* and + allowing one turn. So an emitted call buys nothing: it burns the turn and returns no summary. The + reactive path gets no retry; the manual path retries once outside the fork with a reduced tool + set, then gives up as well. Three + consecutive failures open a circuit breaker that skips later automatic compaction with no API + call, until a successful compaction or a fresh query invocation resets it — which never happens + inside one headless or subagent run, so the context grows until "Prompt is too long". + `isClaudeCodeCompactRequest` keys on the envelope text and nothing else. Two rules it must keep: + **do not re-narrow it to a particular tool** (`StructuredOutput` was the old precondition and + missed every session without a schema — 15 of 173 real translated compact requests in the local + ledgers), and **keep the header match anchored to the start of a text block**, because clodex's + own sources, agent reports and pasted prompts quote the envelope and an unanchored match strips + their tools. Tool *definitions* stay in the request so the cached prompt prefix still matches. - `streamAnthropicResponse` maps SDK events to Anthropic SSE, aborting after 120s without an event. - `modelPrefersResponsesApi()` selects `provider.responses(id)` for models requiring the Responses API (GPT-5.4+, GPT-5.5, `*-codex`, o-series); `provider.chat(id)` otherwise. Originator string is diff --git a/src/sdk-adapter.ts b/src/sdk-adapter.ts index 264a1ba3..c7f6ac1a 100644 --- a/src/sdk-adapter.ts +++ b/src/sdk-adapter.ts @@ -471,27 +471,45 @@ const COMPACT_TEXT_ONLY_START = 'CRITICAL: Respond with TEXT ONLY. Do NOT call a const COMPACT_TEXT_ONLY_END = 'REMINDER: Do NOT call any tools. Respond with plain text only'; /** - * Claude Code's structured-output agents inherit the terminal StructuredOutput - * tool when they fork a reactive compaction turn, even though the compact prompt - * requires plain text and rejects every tool call. OpenAI-family models tend to - * call that highly salient tool, leaving Claude Code with an empty summary. + * Claude Code forks its reactive-compaction turn with the SAME tool definitions + * as an ordinary turn and relies on the prompt alone — "Respond with TEXT ONLY" + * — to stop the model calling them. OpenAI-family models ignore that and answer + * with whichever tool the conversation made salient: StructuredOutput in a + * schema-mode agent, but Bash after a shell-heavy session, and in principle any + * tool at all. The fork denies tool EXECUTION and allows one turn, so a call it + * emits buys nothing: it burns the turn and returns no summary text. The + * reactive path has no second chance at all; the manual path retries once + * outside the fork with a reduced tool set, and then gives up too. Claude + * Code discards the attempt, and three consecutive failures open a circuit + * breaker that short-circuits every later AUTOMATIC compaction — with no API + * call — until the counter is reset by a successful compaction or a fresh query + * invocation. A headless or subagent run is a single invocation, so there it + * never resets: the conversation grows until it dies on Claude Code's own + * "Prompt is too long" guard. * - * Detect only the observed compact envelope. If Claude Code changes it, this + * So key on the compact envelope only — the marker text is what identifies this + * turn, never the tool list, which is why the earlier StructuredOutput + * precondition was too narrow. If Claude Code changes the envelope, this * deliberately fails open rather than stripping tools from an ordinary request. + * + * The envelope must OPEN a text block, not merely appear in one. Every builder + * puts the header first and appends the reminder to the same string, so an + * anchored match costs nothing; an unanchored one fires on any turn that merely + * quotes the envelope — a pasted prompt, a subagent's report, a read of this + * very file — and silently takes tools away from a turn that needed them. */ -function isClaudeCodeStructuredOutputCompactRequest(body: AnthropicRequest): boolean { +function isClaudeCodeCompactRequest(body: AnthropicRequest): boolean { if (body.diagnostics !== undefined) return false; - if (!body.tools?.some(candidate => candidate.name === 'StructuredOutput')) return false; const finalMessage = body.messages.at(-1); if (!finalMessage || finalMessage.role !== 'user') return false; - const text = typeof finalMessage.content === 'string' - ? finalMessage.content + const texts = typeof finalMessage.content === 'string' + ? [finalMessage.content] : finalMessage.content .filter(block => block.type === 'text') - .map(block => block.text ?? '') - .join('\n'); - return text.includes(COMPACT_TEXT_ONLY_START) && text.includes(COMPACT_TEXT_ONLY_END); + .map(block => block.text ?? ''); + return texts.some(text => + text.startsWith(COMPACT_TEXT_ONLY_START) && text.includes(COMPACT_TEXT_ONLY_END)); } export function translateRequest( @@ -519,7 +537,7 @@ export function translateRequest( // minimal request shapes, so cast at this boundary. Keep compact-request tool // definitions intact for prompt-cache prefix reuse; toolChoice='none' below // makes them unavailable at the provider API rather than by prompt compliance. - const compactRequest = isClaudeCodeStructuredOutputCompactRequest(body); + const compactRequest = isClaudeCodeCompactRequest(body); let upstreamTools = resolveUpstreamTools( body.tools as unknown as AnthropicToolDefinition[] | undefined, messages as unknown as AnthropicRequestMessage[], diff --git a/tests/sdk-adapter.test.ts b/tests/sdk-adapter.test.ts index aa003baa..c512258a 100644 --- a/tests/sdk-adapter.test.ts +++ b/tests/sdk-adapter.test.ts @@ -683,6 +683,272 @@ describe('translateRequest', () => { expect(compact.providerOptions?.openai?.promptCacheKey) .toBe(ordinary.providerOptions?.openai?.promptCacheKey); }); + + // Head and tail verbatim from the Claude Code 2.1.259 bundle (the summary + // instructions between them are elided). Claude Code wraps EVERY compaction + // turn in these, whatever tools the session happens to be carrying. + const CC_COMPACT_HEAD = [ + 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.', + '', + '- Do NOT use Read, Bash, Grep, Glob, Edit, Write, or ANY other tool.', + '- You already have all the context you need in the conversation above.', + '- Tool calls will be REJECTED and will waste your only turn — you will fail the task.', + '- Your entire response must be plain text: an block followed by a block.', + ].join('\n'); + const CC_COMPACT_TAIL = '\n\nREMINDER: Do NOT call any tools. Respond with plain text only — ' + + 'an block followed by a block. Tool calls will be rejected and you will ' + + 'fail the task.'; + const ccCompactPrompt = `${CC_COMPACT_HEAD}\n\nYour task is to create a detailed summary of the ` + + `conversation so far.${CC_COMPACT_TAIL}`; + // A shell-heavy session: no StructuredOutput anywhere, which is the normal + // shape for an interactive session and for a schema-less workflow agent. + const shellSessionTools = [ + { name: 'Bash', input_schema: { type: 'object' } }, + { name: 'Read', input_schema: { type: 'object' } }, + { name: 'Grep', input_schema: { type: 'object' } }, + ]; + + it('disables tools for a compact request from a session that has no StructuredOutput tool', () => { + // The dominant shape on the wire: Claude Code merges the compact prompt into + // the preceding tool_result turn, so the final user message is + // [tool_result, text]. 193 of 194 real compact requests in the local + // diagnostics ledgers arrive exactly like this. + const blockContent = translateRequest({ + model: 'gpt-5.6-sol', + messages: [ + { role: 'user', content: 'run the build' }, + { + role: 'assistant', + content: [{ type: 'tool_use', id: 'call_133', name: 'Bash', input: { command: 'make' } }], + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call_133', content: 'build output' }, + { type: 'text', text: ccCompactPrompt }, + ], + }, + ], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + + expect(blockContent.toolChoice).toBe('none'); + // Definitions stay so the cached prompt prefix still matches. + expect(blockContent.tools && Object.keys(blockContent.tools)).toEqual(['Bash', 'Read', 'Grep']); + + // The other observed shape (1 of 194): another text block precedes the + // prompt. The envelope has to be found per block, not in the joined text — + // and Claude Code appends reminder blocks after it as well, so neither the + // first nor the last block can be the only one inspected. + for (const content of [ + [ + { type: 'text', text: 'Plan mode is active.' }, + { type: 'text', text: ccCompactPrompt }, + ], + [ + { type: 'tool_result', tool_use_id: 'call_1', content: 'build output' }, + { type: 'text', text: `${ccCompactPrompt}\n` }, + { type: 'text', text: 'Background task finished.' }, + ], + ]) { + const siblingBlocks = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ role: 'user', content }], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(siblingBlocks.toolChoice).toBe('none'); + } + + // Claude Code builds the compact prompt as a single string message; assert + // the plain-string arrival shape too, not only the block-array one. + const stringContent = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ role: 'user', content: ccCompactPrompt }], + tools: shellSessionTools, + tool_choice: { type: 'any' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + + expect(stringContent.toolChoice).toBe('none'); + expect(stringContent.tools && Object.keys(stringContent.tools)).toEqual(['Bash', 'Read', 'Grep']); + + // The manual /compact builder emits the same envelope around a different + // body, and its "Additional Instructions" variant appends after the body, + // before the reminder. Both must still be recognised. + for (const variant of [ + `${CC_COMPACT_HEAD}\n\nSummarize the conversation up to the selected message.${CC_COMPACT_TAIL}`, + `${CC_COMPACT_HEAD}\n\nYour task is to create a detailed summary.` + + `\n\nAdditional Instructions:\nfocus on the auth work${CC_COMPACT_TAIL}`, + ]) { + const manual = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ role: 'user', content: variant }], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(manual.toolChoice).toBe('none'); + } + }); + + it('ignores a compact envelope that is quoted rather than issued', () => { + // Claude Code's prompt always OPENS its text block. Everything below merely + // contains the envelope: a user pasting it, an agent's report repeating it, + // and a file read whose content is this repo's own source. Each must keep + // its tools — the last one is why the envelope has to be anchored, since + // clodex's own sources carry both markers verbatim. + const quoted = [ + `Why does Claude Code send this?\n\n${ccCompactPrompt}\n\nCheck the bundle and tell me.`, + `[agent report] I verified the envelope. It reads:\n${ccCompactPrompt}\nBoth markers matched.`, + ]; + for (const content of quoted) { + const params = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ role: 'user', content }], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(params.toolChoice).toBe('auto'); + } + + // Same text arriving as a text block alongside a tool_result, which is the + // shape a file read or a subagent result actually takes. + const quotedBlock = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call_1', content: 'ok' }, + { type: 'text', text: `Here is what the file said:\n${ccCompactPrompt}` }, + ], + }], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(quotedBlock.toolChoice).toBe('auto'); + + // The tool_result channel is deliberately not searched at all: a Read or a + // Bash `cat` of a file holding the envelope must never disarm the turn. + for (const content of [ + ccCompactPrompt, + [{ type: 'text', text: ccCompactPrompt }] as unknown as string, + ]) { + const viaToolResult = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call_1', content }, + { type: 'text', text: 'summarize that file for me' }, + ], + }], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(viaToolResult.toolChoice).toBe('auto'); + } + + // Both markers must be in the SAME block. Claude Code builds one string, so + // header in one block and reminder in another is someone else's text. + const splitAcrossBlocks = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ + role: 'user', + content: [ + { type: 'text', text: `${CC_COMPACT_HEAD}\n\nis what it opens with.` }, + { type: 'text', text: `and it closes with${CC_COMPACT_TAIL}` }, + ], + }], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(splitAcrossBlocks.toolChoice).toBe('auto'); + + // Near misses. Each marker has to stay a whole sentence: a message that + // merely opens the same way, or that ends with the same phrase, is ordinary + // prose and must keep its tools. + const nearMisses = [ + 'Respond with TEXT ONLY where you can, and keep to plain text only if possible.', + // Opens like the header, then diverges — only a truncated header matches. + `CRITICAL: Respond with TEXT ONLY when you summarise, but run the tests first.` + + `${CC_COMPACT_TAIL}`, + // Real header, but the closing reminder is ordinary prose. + `${CC_COMPACT_HEAD}\n\nSummarise the work.\n\nAnswer in plain text only.`, + ]; + for (const content of nearMisses) { + const params = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ role: 'user', content }], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(params.toolChoice).toBe('auto'); + } + }); + + it('leaves tool choice alone on an ordinary turn carrying the same tools', () => { + const auto = translateRequest({ + model: 'gpt-5.6-sol', + messages: [ + { role: 'user', content: 'summarize the conversation so far, then keep going' }, + ], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(auto.toolChoice).toBe('auto'); + + const required = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ role: 'user', content: 'do not call any tools, just answer' }], + tools: shellSessionTools, + tool_choice: { type: 'any' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(required.toolChoice).toBe('required'); + + const named = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ role: 'user', content: 'read the file' }], + tools: shellSessionTools, + tool_choice: { type: 'tool', name: 'Read' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(named.toolChoice).toEqual({ type: 'tool', toolName: 'Read' }); + + // Both markers are required. The existing structured-output case pins the + // head-only half; this pins the tail-only half. + const tailOnly = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ role: 'user', content: `quote it back to me:${CC_COMPACT_TAIL}` }], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(tailOnly.toolChoice).toBe('auto'); + + // A trailing assistant prefill is not a compact turn, whatever it contains. + const prefill = translateRequest({ + model: 'gpt-5.6-sol', + messages: [ + { role: 'user', content: 'echo the compaction preamble' }, + { role: 'assistant', content: ccCompactPrompt }, + ], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(prefill.toolChoice).toBe('auto'); + + // The markers only count on the FINAL user message — an earlier compact + // envelope left in history must not disarm a later real turn. + const historic = translateRequest({ + model: 'gpt-5.6-sol', + messages: [ + { role: 'user', content: ccCompactPrompt }, + { role: 'assistant', content: 'earlier work' }, + { role: 'user', content: 'now finish the refactor' }, + ], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true }); + expect(historic.toolChoice).toBe('auto'); + }); }); describe('generateAnthropicResponse', () => { From e5ad18a4534955f3f17df8e58414eebc0d3b541e Mon Sep 17 00:00:00 2001 From: integ Date: Fri, 4 Sep 2026 02:18:30 -0500 Subject: [PATCH 2/2] feat(compaction): warn when a Claude Code update may stop long sessions compacting --- .claude/docs/patcher.md | 38 ++- .claude/docs/translation.md | 11 + canary/clodex-patch-canary-platforms.sh | 198 +++++++++++--- canary/clodex-patch-canary-selftest.sh | 253 ++++++++++++++++- canary/clodex-patch-canary.sh | 152 +++++++---- scripts/probe-patch-mechanism.mjs | 21 ++ src/claude-code-compact-prompt.ts | 38 +++ src/proxy.ts | 1 + src/sdk-adapter.ts | 87 +++++- src/server/router.ts | 1 + tests/probe-patch-mechanism.test.ts | 102 +++++++ tests/probe-patch-sites.test.ts | 46 +++- tests/proxy.test.ts | 54 ++++ tests/sdk-adapter.test.ts | 349 ++++++++++++++++++++++++ tests/server-router.test.ts | 54 +++- 15 files changed, 1288 insertions(+), 117 deletions(-) create mode 100644 src/claude-code-compact-prompt.ts create mode 100644 tests/probe-patch-mechanism.test.ts diff --git a/.claude/docs/patcher.md b/.claude/docs/patcher.md index 6739eb40..3f8eae87 100644 --- a/.claude/docs/patcher.md +++ b/.claude/docs/patcher.md @@ -340,25 +340,33 @@ tweakcc's own repack reads back as an ordinary module name. back carrying what was written. **It also applies every patch site to that build's own bundle** (`scripts/probe-patch-sites.mjs`, - which calls the real `applyClodexPatches` with a synthetic config that activates all of them), - fails on any `FAIL`, `SKIP`, missing or duplicated site, and publishes **what the transforms - produced** rather than the pristine bytes — so the byte-for-byte readback also proves clodex's own - emitted patch survives the PE/ELF/Mach-O round trip. Read that precisely: since the write stopped - rebuilding the blob, the patched bytes never enter tweakcc's repack, so what the readback proves - is that the container RESIZE works on this format and that clodex's own publish round-trips. What - pins the sizing arithmetic against the real repack is the probe's `blob-sized-as-planned` check. Anchors were assumed platform-independent - until Claude Code 2.1.238, where `PATCH 5: model picker options` matched five builds and missed - `linux-arm64`, `linux-arm64-musl` and `win32-arm64`. + which calls the real `applyClodexPatches` with a synthetic config that activates all of them), and + checks that the exact Claude Code compaction markers used by the runtime text-only guard still + occur in that platform's extracted JavaScript. Both consumers import the same marker module. The + hourly canary runs this probe against every downloaded platform build before any stronger host or + container execution check. The probe fails on a missing marker or any `FAIL`, `SKIP`, missing or + duplicated patch site, and publishes **what the transforms produced** rather than the pristine + bytes — so the byte-for-byte readback also proves clodex's own emitted patch survives the + PE/ELF/Mach-O round trip. Read that precisely: since the write stopped rebuilding the blob, the + patched bytes never enter tweakcc's repack, so what the readback proves is that the container + RESIZE works on this format and that clodex's own publish round-trips. The + `blob-sized-as-planned` check pins the sizing arithmetic against the real repack. Anchors were + assumed platform-independent until Claude Code 2.1.238, where `PATCH 5: model picker options` + matched five builds and missed `linux-arm64`, `linux-arm64-musl` and `win32-arm64`. Two things it still cannot tell you: **that the patched binary runs**, and **that an anchor bound to the function it was aimed at** rather than a lookalike that also emits valid JavaScript. Only a host or containerised `clodex patch` answers the first. `clodex patch` resolves the version by executing the binary (`getClaudeVersionForBinary`), which is exactly why a foreign binary can go - through the probe and not through the real command. + through the probe and not through the real command. A `compact-prompt-markers` failure is not a + broken patch site. On a complete extraction it means Claude Code changed the prompt wording and + the text-only guard stopped matching; first confirm the bundle reader still exposes both builders, + then update `src/claude-code-compact-prompt.ts` from their extracted wording. Presence catches + removal or in-place rewording of today's strings, not a new builder that leaves both old strings. `tests/probe-patch-sites.test.ts` pins the probe's synthetic config and its expected-site list against the real transform set — so a new or renamed `PATCH` site reddens `pnpm test` rather than - failing five platforms in the hourly canary. Revisit it whenever you bump + failing the hourly canary on every downloaded platform build. Revisit it whenever you bump `PATCH_TRANSFORMS_VERSION`. ## Patcher invariants @@ -391,10 +399,10 @@ tweakcc's own repack reads back as an ordinary module name. because the twin was inserted into exactly that gap. Note what the replacement does and does not prove: it validates the appender the builder loops through, so a different caller of the genuine appender would inherit that evidence; it rejects the realistic impostor, which brings its own. - Verify with the real-bundle - harness over **every platform's** bundle (`scripts/extract-cc-bundles.mjs` reads a foreign binary - fine), not just this Mac's: the canary's probe legs exercise zero patch sites, so win32-arm64 - recorded a `pass` for the release this broke. + Verify with the real-bundle harness over **every downloaded platform's** bundle + (`scripts/extract-cc-bundles.mjs` reads a foreign binary fine), not just this Mac's. This is why the + canary now applies the patch sites to every bundle it downloads: before that check existed, + win32-arm64 recorded a `pass` for the release this broke. - **An anchor that spells out a statement upstream can delete is a required patch waiting to fail.** Claude Code 2.1.239 moved BOTH ends of PATCH 10's child-env builder in a single release — the opening `let` gained an optional call and a *destructuring* declarator (`{settingsColorEnv:n}=e`, diff --git a/.claude/docs/translation.md b/.claude/docs/translation.md index 591d83f7..8cb3a2a0 100644 --- a/.claude/docs/translation.md +++ b/.claude/docs/translation.md @@ -37,6 +37,17 @@ hand-rolled per-provider translation. Preserved hard-won behavior: ledgers), and **keep the header match anchored to the start of a text block**, because clodex's own sources, agent reports and pasted prompts quote the envelope and an unanchored match strips their tools. Tool *definitions* stay in the request so the cached prompt prefix still matches. + If the strict header changes, a deliberately bounded warning-only recognizer can report one + subset of drift without changing tool choice: after an optional known severity label, the new + header must still start with `respond`, `return`, `answer`, `output`, `write`, or `provide`, then + say text only and prohibit tools on one short line; the rejected-tool/only-turn anchor must remain + at line start. It is not a general drift detector. The reverse shape — strict header + intact, reminder changed — is deliberately invisible at runtime because it is indistinguishable + from a pasted header. The per-build probe checks both strict markers in every extracted bundle; + that catches their removal or in-place rewording, not a new third builder that leaves both old + strings present. A warning is diagnostic only: tools stay enabled and compaction can still fail + until clodex updates its markers. Terminal notices are capped at three `cc_version` signatures per + process (plus one suppression line), while every sighting remains in the trace log. - `streamAnthropicResponse` maps SDK events to Anthropic SSE, aborting after 120s without an event. - `modelPrefersResponsesApi()` selects `provider.responses(id)` for models requiring the Responses API (GPT-5.4+, GPT-5.5, `*-codex`, o-series); `provider.chat(id)` otherwise. Originator string is diff --git a/canary/clodex-patch-canary-platforms.sh b/canary/clodex-patch-canary-platforms.sh index d207229a..d74ace2a 100755 --- a/canary/clodex-patch-canary-platforms.sh +++ b/canary/clodex-patch-canary-platforms.sh @@ -22,21 +22,25 @@ # across two clodex releases while macOS stayed green, because the canary only # ever tested the host. An outside contributor reported it, not us. # -# So every published platform is now tested, by the strongest means available for it: +# So every downloaded platform build now runs the cross-platform probe, plus the strongest +# execution check available for it: # -# host the real `clodex patch` on this Mac — the authoritative "is it safe for me to -# update" answer, and the only leg that patches AND then starts the binary here. -# container the real `clodex patch` inside a native-architecture Linux container. `clodex -# patch` resolves the version by EXECUTING the binary, so a foreign binary cannot go -# through the real command on the host — a container is the only way to run the whole -# thing against ELF. # probe scripts/probe-patch-mechanism.mjs from the clodex checkout under test: it applies -# every clodex patch site to the bundle it extracts from THAT build, repacks what -# those transforms produced, and runs the same shim/read/repack/restore cycle the -# patcher runs. It never executes the binary, so it works from macOS against every -# format, and it is the only coverage win32 and linux-x64 can have cheaply. -# What it cannot show: that the patched binary starts, and that an anchor bound to -# the function it was meant to bind to rather than a lookalike that also parses. +# every clodex patch site to the bundle it extracts from THAT build, checks the exact +# compaction-prompt markers, repacks what those transforms produced, and runs the same +# shim/read/repack/restore cycle the patcher runs. It never executes the binary, so it +# works from macOS against every format. +# host the real `clodex patch` on this Mac, after the probe — the authoritative "is it safe +# for me to update" answer, and the only leg that patches AND then starts the binary +# here. +# container the real `clodex patch` inside a native-architecture Linux container, after the +# probe. `clodex patch` resolves the version by EXECUTING the binary, so a foreign +# binary cannot go through the real command on the host — a container is the only way +# to run the whole thing against ELF. +# +# On probe-only platforms, what remains unproved is that the patched binary starts. On every +# platform, matching an anchor cannot prove it bound to the intended function rather than a +# lookalike that also parses. # # A leg that cannot RUN (no Docker, a platform package that has not published yet, a container # that could not install its dependencies) is an `error`, never a `fail`. Only evidence that the @@ -147,7 +151,7 @@ docker_ready() { [ "$DOCKER_READY" = "yes" ] } -# The strongest leg this run can actually run for a platform. +# The matrix mode: strongest execution tier, or `probe` when none can run. platform_mode() { # platform_mode if [ "$1" = "$(host_platform)" ]; then printf 'host'; return 0; fi if [ -n "$(platform_image "$1")" ] && docker_ready; then printf 'container'; return 0; fi @@ -213,10 +217,10 @@ fetch_platform_binary() { # fetch_platform_binary # --------------------------------------------------------------------- the legs # -# Every leg leaves the same four globals behind, so one evaluator serves all three: +# Every leg leaves the same globals behind, so matrix assembly is shared across all modes: # LEG_STATUS pass | fail | error # LEG_REASONS why it failed, one "- " line each (empty on pass) -# LEG_SITES per-site OK/SKIP/FAIL map as JSON ({} when the leg does not run patch sites) +# LEG_SITES named probe checks plus patch-site OK/SKIP/FAIL results as JSON # LEG_MARKERS ccpatch: markers found in the published binary, as JSON ([] likewise) # plus LEG_DETAIL, a JSON object of whatever else is worth recording for that leg. @@ -460,23 +464,23 @@ run_leg_probe() { # run_leg_probe " return 0 fi - # Fenced like the other two legs. The probe redirects TWEAKCC_CONFIG_DIR itself only when it is - # UNSET, so an operator with it exported in their shell would otherwise have the probe write - # into their real ~/.tweakcc — the one thing every leg here promises never to touch. - mkdir -p "$WORK/$platform/tweakcc" + # Fenced like the other two legs. Keep its tweakcc state separate from a later container patch: + # the probe does not write there today, but sharing makes a future tweakcc cache or backup alter + # the supposedly fresh execution tier. An exported operator value must never reach ~/.tweakcc. + mkdir -p "$WORK/$platform/probe-tweakcc" set +e ( cd "$REPO_DIR" && with_timeout "$CANARY_PATCH_TIMEOUT" \ env -u CLODEX_TRACE -u FORCE_COLOR -u CLICOLOR_FORCE \ HOME="$WORK/home" \ CLODEX_HOME="$WORK/clodex-home" \ - TWEAKCC_CONFIG_DIR="$WORK/$platform/tweakcc" \ + TWEAKCC_CONFIG_DIR="$WORK/$platform/probe-tweakcc" \ "$NODE_BIN_DIR/node" "$REPO_DIR/scripts/probe-patch-mechanism.mjs" "$PLATFORM_BINARY" \ --label "$platform" --expect-version "$version" --json \ --scratch "$WORK/$platform/probe-scratch" ) \ > "$json" 2> "$WORK/$platform/probe.stderr" rc=$? set -e - rm -rf "$WORK/$platform/probe-scratch" + rm -rf "$WORK/$platform/probe-scratch" "$WORK/$platform/probe-tweakcc" evaluate_probe_leg "$platform" "$json" "$rc" "$WORK/$platform/probe.stderr" return 0 @@ -513,6 +517,17 @@ evaluate_probe_leg() { # evaluate_probe_leg return 0 fi + # Host and container execution says whether the patcher runs; it cannot notice that Claude Code + # reworded the compaction prompt and silently disabled the runtime's exact text-only guard. Every + # platform therefore needs this probe check even when a stronger execution leg follows it. + if ! jq -e 'any(.checks[]; .name == "compact-prompt-markers" and (.ok | type) == "boolean")' \ + "$json" >/dev/null 2>&1; then + LEG_STATUS=error + LEG_REASONS="- the $platform probe reported no compact-prompt-markers result, so prompt drift was not checked — the clodex build under test (${MAIN_SHA:-unknown} on $CANARY_BRANCH) predates compaction-marker checking +" + return 0 + fi + LEG_DETAIL="$(jq -c '{format, entryState, shimUsed, pristineSize, publishedSize, growth, detectedVersion, sourceBytes, durationMs, patchSites: (.patchSiteSummary // null)}' "$json")" # Both sets of names, in one map, because both are checks that must keep holding: the mechanism @@ -532,10 +547,78 @@ evaluate_probe_leg() { # evaluate_probe_leg " fi failed_sites="$(jq -r '[(.patchSites // [])[] | select(.status == "FAIL") | .name] | join("; ")' "$json")" - log "$platform probe: $(jq -r '.verdict' "$json") ($(jq -r '(.durationMs/1000|floor)' "$json")s, $(jq -r '.format' "$json"), $(jq -r '.patchSiteSummary.applied // 0') patch sites applied${failed_sites:+, FAILED: $failed_sites})" + log "$platform probe: $(jq -r '.verdict' "$json") ($(jq -r '(.durationMs/1000|floor)' "$json")s, $(jq -r '.format' "$json"), $(jq -r '.patchSiteSummary.applied // 0' "$json") patch sites applied${failed_sites:+, FAILED: $failed_sites})" return 0 } +# Keep an infrastructure failure identifiable when a release failure from the other tier decides +# the merged row's status. matrix_reason_groups must not present that line as release evidence. +append_merged_leg_reasons() { # append_merged_leg_reasons + local status="$1" reasons="$2" line + [ "$status" != "pass" ] || return 0 + while IFS= read -r line; do + [ -n "$line" ] || continue + if [ "$status" = "error" ]; then + LEG_REASONS="${LEG_REASONS}- CANARY INFRASTRUCTURE: ${line#- }"$'\n' + else + LEG_REASONS="${LEG_REASONS}${line}"$'\n' + fi + done <<< "$reasons" +} + +# Fold the mandatory cross-platform probe into a host/container execution result. A real release +# failure decides the status ahead of an infrastructure error, while both tiers' reasons survive. +merge_probe_with_execution() { # merge_probe_with_execution + local probe_status="$1" probe_reasons="$2" probe_sites="$3" probe_markers="$4" probe_detail="$5" + local execution_status="$LEG_STATUS" execution_reasons="$LEG_REASONS" + + # Both tiers report PATCH names. Preserve each answer rather than letting a successful real patch + # overwrite a synthetic all-sites failure (or vice versa); mechanism checks have no collision. + LEG_SITES="$(jq -n -c --argjson probe "$probe_sites" --argjson execution "$LEG_SITES" ' + ($probe | with_entries(if (.key | startswith("PATCH ")) + then .key = ("probe:" + .key) else . end)) + $execution')" + LEG_MARKERS="$(jq -n -c --argjson probe "$probe_markers" --argjson execution "$LEG_MARKERS" \ + '$probe + $execution | unique')" + LEG_DETAIL="$(jq -n -c --argjson probe "$probe_detail" --argjson execution "$LEG_DETAIL" \ + '$execution + {probe: $probe}')" + + if [ "$probe_status" = "fail" ] || [ "$execution_status" = "fail" ]; then + LEG_STATUS=fail + elif [ "$probe_status" = "error" ] || [ "$execution_status" = "error" ]; then + LEG_STATUS=error + else + LEG_STATUS=pass + fi + LEG_REASONS="" + append_merged_leg_reasons "$probe_status" "$probe_reasons" + append_merged_leg_reasons "$execution_status" "$execution_reasons" +} + +# Probe every downloaded build before a host/container leg mutates it. On platforms that cannot be +# executed here, the probe remains the whole leg. An unknown execution mode dies here rather than +# bypassing the per-build prompt-marker check quietly. +run_platform_leg() { # run_platform_leg + local platform="$1" version="$2" mode="$3" + local probe_status probe_reasons probe_sites probe_markers probe_detail + + run_leg_probe "$platform" "$version" + [ "$mode" != "probe" ] || return 0 + + probe_status="$LEG_STATUS" + probe_reasons="$LEG_REASONS" + probe_sites="$LEG_SITES" + probe_markers="$LEG_MARKERS" + probe_detail="$LEG_DETAIL" + + case "$mode" in + host) run_leg_host "$platform" "$version" ;; + container) run_leg_container "$platform" "$version" ;; + *) die "unknown platform mode: $mode" ;; + esac + merge_probe_with_execution \ + "$probe_status" "$probe_reasons" "$probe_sites" "$probe_markers" "$probe_detail" +} + # --------------------------------------------------------------- the whole matrix # Run every platform and leave one JSON object per line in $WORK/matrix.jsonl. @@ -585,11 +668,7 @@ run_platform_matrix() { # run_platform_matrix " mode=none else - case "$mode" in - host) run_leg_host "$platform" "$version" ;; - container) run_leg_container "$platform" "$version" ;; - *) run_leg_probe "$platform" "$version" ;; - esac + run_platform_leg "$platform" "$version" "$mode" fi # A platform that should have had a full containerised patch but got the probe instead was @@ -641,18 +720,43 @@ matrix_status_of() { # matrix_status_of matrix_reason_groups() { jq -r -s ' map(select(.status == "fail")) - | map({platform, reasons: (.reasons | split("\n") | map(select(length > 0)))}) + | map({platform, reasons: (.reasons | split("\n") + | map(select(length > 0)) + | map(select(startswith("- CANARY INFRASTRUCTURE: ") | not)))}) | map(.reasons[] as $r | {platform, r: $r}) | group_by(.r) | map("\(.[0].r) [\(map(.platform) | join(", "))]") | join("\n")' "$MATRIX" } -# The same, for legs that could not be run at all. +# A prompt-marker miss breaks long OpenAI sessions, but it is not evidence that binary patching +# broke. Keep that category distinct only when every release-origin failure is the marker check; +# tagged infrastructure failures may coexist without changing the diagnosis. +matrix_failures_are_compact_prompt_drift_only() { + jq -e -s ' + [ .[] | select(.status == "fail") ] as $failed + | ($failed | length) > 0 + and all($failed[]; + ((.sites // {})["compact-prompt-markers"] == "FAIL") + and ([((.sites // {}) | to_entries[]) + | select(.value == "FAIL" and .key != "compact-prompt-markers")] | length) == 0 + and ([.reasons | split("\n")[] + | select(length > 0) + | select(startswith("- CANARY INFRASTRUCTURE: ") | not) + | select(startswith("- Compaction-prompt drift is not a patch failure:") | not)] + | length) == 0) + ' "$MATRIX" >/dev/null +} + +# Infrastructure reasons from an `error` row, plus tagged infrastructure reasons retained in a +# `fail` row whose other tier found a release defect. matrix_error_notes() { jq -r -s ' - map(select(.status == "error")) - | map(.reasons | split("\n") | map(select(length > 0))[]) + map(. as $row + | ($row.reasons | split("\n") | map(select(length > 0))[]) + | select($row.status == "error" or startswith("- CANARY INFRASTRUCTURE: ")) + | sub("^- CANARY INFRASTRUCTURE: "; "- ")) + | unique | join("\n")' "$MATRIX" } @@ -674,9 +778,17 @@ matrix_coverage_line() { def sym: if .status == "pass" and .downgraded then ":warning:" elif .status == "pass" then ":white_check_mark:" elif .status == "fail" then ":x:" else ":grey_question:" end; + def marker_failed: + .status == "fail" and ((.sites // {})["compact-prompt-markers"] == "FAIL"); def how: if .status != "pass" and .status != "fail" then "NOT TESTED" - elif .mode == "host" then "full patch, this Mac" - elif .mode == "container" then "full patch in \(.detail.image // "a container")" + elif marker_failed and .mode == "host" then + "compaction prompt marker check failed; full patch also ran on this Mac" + elif marker_failed and .mode == "container" then + "compaction prompt marker check failed; full patch also ran in \(.detail.image // "a container")" + elif marker_failed and .mode == "probe" then + "compaction prompt marker check failed; native execution not checked" + elif .mode == "host" then "bundle probe + full patch, this Mac" + elif .mode == "container" then "bundle probe + full patch in \(.detail.image // "a container")" elif .mode == "probe" and .downgraded then "bundle patch + binary handling — no container, so the patched binary was never started here" elif .mode == "probe" then @@ -690,8 +802,16 @@ matrix_coverage_line() { matrix_coverage_brief() { jq -r -s ' def sym: if .status == "fail" then ":x:" else ":grey_question:" end; - def how: if .mode == "host" then "full patch, this Mac" - elif .mode == "container" then "full patch in \(.detail.image // "a container")" + def marker_failed: + .status == "fail" and ((.sites // {})["compact-prompt-markers"] == "FAIL"); + def how: if marker_failed and .mode == "host" then + "compaction prompt marker check failed; full patch also ran on this Mac" + elif marker_failed and .mode == "container" then + "compaction prompt marker check failed; full patch also ran in \(.detail.image // "a container")" + elif marker_failed and .mode == "probe" then + "compaction prompt marker check failed; native execution not checked" + elif .mode == "host" then "bundle probe + full patch, this Mac" + elif .mode == "container" then "bundle probe + full patch in \(.detail.image // "a container")" elif .mode == "probe" then "bundle patch + binary handling" else "not tested" end; (map(select(.status != "pass")) | map("\(sym) *\(.platform)* — \(how)")) as $bad @@ -727,8 +847,10 @@ matrix_mode_of() { # matrix_mode_of matrix_applied_sites() { # matrix_applied_sites jq -r -s --arg p "$1" \ 'map(select(.platform == $p) | .sites) | last // {} - | [to_entries[] | select(.value == "OK")] | length' "$MATRIX" + | [to_entries[] | select(.key | startswith("PATCH ")) | select(.value == "OK")] | length' "$MATRIX" } matrix_total_sites() { # matrix_total_sites - jq -r -s --arg p "$1" 'map(select(.platform == $p) | .sites | length) | last // 0' "$MATRIX" + jq -r -s --arg p "$1" \ + 'map(select(.platform == $p) | .sites) | last // {} + | [to_entries[] | select(.key | startswith("PATCH "))] | length' "$MATRIX" } diff --git a/canary/clodex-patch-canary-selftest.sh b/canary/clodex-patch-canary-selftest.sh index 7a2f5a52..4f951f85 100755 --- a/canary/clodex-patch-canary-selftest.sh +++ b/canary/clodex-patch-canary-selftest.sh @@ -378,10 +378,11 @@ gate_denied "a subset run never counts as full coverage" \ # 17. A SKIP is a site that did nothing. Counting it as applied is how "all 11 applied" stays # true while a clodex feature is quietly off. -MATRIX_SITES='{"PATCH 1: a":"OK","PATCH 3: b":"OK","PATCH 7: c":"SKIP","PATCH 9: d":"OK"}' \ +MATRIX_SITES='{"compact-prompt-markers":"OK","published-content":"OK","PATCH 1: a":"OK","PATCH 3: b":"OK","PATCH 7: c":"SKIP","PATCH 9: d":"OK"}' \ write_matrix "$HOST:host:pass" -matrix_check "applied counts OK only, not SKIP" "3" "$(matrix_applied_sites "$HOST")" -matrix_check "reported counts every site" "4" "$(matrix_total_sites "$HOST")" +matrix_check "applied counts OK patch sites only, not SKIP or probe checks" "3" \ + "$(matrix_applied_sites "$HOST")" +matrix_check "reported counts patch sites only, not probe checks" "4" "$(matrix_total_sites "$HOST")" matrix_check "a platform with no sites reports zero" "0" "$(matrix_applied_sites nonesuch)" # 18. The brief coverage block used in the fail alert: failures in full, passes collapsed. @@ -432,6 +433,7 @@ matrix_check "--no-container is not reported as Docker being down" "0" \ # a clean pass while reporting the same break on the two Linux builds that DO have images. MECHANISM_CHECKS='[{"name":"pristine-parses","ok":true,"detail":"entry module is needs-shim"}, {"name":"read-content","ok":true,"detail":"28147627 bytes of JavaScript"}, + {"name":"compact-prompt-markers","ok":true,"detail":"both strict compaction prompt markers are present"}, {"name":"published-content","ok":true,"detail":"byte-for-byte"}]' # write_probe_json [reasons-json] [extra-check-json] @@ -469,6 +471,20 @@ matrix_check "a probe records its patch sites alongside its mechanism checks" "O matrix_check "a probe records how many patch sites applied" "3" \ "$(printf '%s' "$LEG_DETAIL" | jq -r '.patchSites.applied')" +# A host/container patch can pass without observing prompt drift, so an older probe that omits the +# exact-marker result cannot be accepted as the prerequisite for either execution leg. +jq 'del(.checks[] | select(.name == "compact-prompt-markers"))' \ + "$TMP/probe-ok.json" > "$TMP/probe-no-compact-markers.json" +leg_reset +evaluate_probe_leg win32-arm64 "$TMP/probe-no-compact-markers.json" 0 "$TMP/probe.stderr" >/dev/null +matrix_check "a probe with no compaction-marker result is never a pass" "error" "$LEG_STATUS" +case "$LEG_REASONS" in + *"prompt drift was not checked"*) + pass=$((pass + 1)); printf 'ok %s\n' "a probe with no compaction-marker result names the coverage gap" ;; + *) + fail=$((fail + 1)); printf 'FAIL a probe with no compaction-marker result says why — got: %s\n' "$LEG_REASONS" ;; +esac + leg_reset write_probe_json "$TMP/probe-p5.json" fail "$PATCH5_MISSING" \ '["patch sites FAILED: PATCH 5: model picker options"]' \ @@ -493,7 +509,236 @@ case "$LEG_REASONS" in *) fail=$((fail + 1)); printf 'FAIL a probe with no patch-site results says why — got: %s\n' "$LEG_REASONS" ;; esac -# 19c. The whole incident, end to end at the matrix level: 2.1.238 lost PATCH 5 on the two arm64 +# 19c. Every mode runs the probe. Host/container then add execution evidence without replacing the +# probe's marker verdict. Stubs drive the real dispatcher and merger; no binary or Docker is needed. +exercise_leg_dispatch() { # exercise_leg_dispatch [probe-status] [execution-status] + local mode="$1" STUB_PROBE_STATUS="${2:-pass}" STUB_EXECUTION_STATUS="${3:-pass}" + ( + CALLS="" + run_leg_probe() { + CALLS="${CALLS:+$CALLS,}probe" + leg_reset + LEG_STATUS="$STUB_PROBE_STATUS" + case "$STUB_PROBE_STATUS" in + pass) LEG_REASONS="" ;; + fail) LEG_REASONS="- compact prompt marker(s) missing: end" ;; + error) LEG_REASONS="- the probe produced no usable result" ;; + esac + LEG_SITES='{"compact-prompt-markers":"FAIL","published-content":"OK","PATCH 10: child network environment":"FAIL"}' + [ "$STUB_PROBE_STATUS" != "pass" ] || LEG_SITES='{"compact-prompt-markers":"OK","published-content":"OK","PATCH 10: child network environment":"OK"}' + LEG_MARKERS='["ccpatch:probe"]' + LEG_DETAIL='{"format":"elf","sourceBytes":28147627}' + } + run_leg_host() { + CALLS="${CALLS:+$CALLS,}host" + leg_reset + LEG_STATUS="$STUB_EXECUTION_STATUS" + LEG_REASONS="$( [ "$STUB_EXECUTION_STATUS" = "pass" ] || printf '%s\n' '- host patch failed' )" + LEG_SITES='{"PATCH 10: child network environment":"OK"}' + LEG_MARKERS='["ccpatch:child-network-env"]' + LEG_DETAIL='{"patchedVersion":"2.1.261"}' + } + run_leg_container() { + CALLS="${CALLS:+$CALLS,}container" + leg_reset + LEG_STATUS="$STUB_EXECUTION_STATUS" + LEG_REASONS="$( [ "$STUB_EXECUTION_STATUS" = "pass" ] || printf '%s\n' '- container patch failed' )" + LEG_SITES='{"PATCH 10: child network environment":"OK"}' + LEG_MARKERS='["ccpatch:child-network-env"]' + LEG_DETAIL='{"image":"node:24-bookworm"}' + } + + run_platform_leg test-platform 2.1.261 "$mode" + jq -n -c --arg calls "$CALLS" --arg status "$LEG_STATUS" --arg reasons "$LEG_REASONS" \ + --argjson sites "$LEG_SITES" --argjson markers "$LEG_MARKERS" --argjson detail "$LEG_DETAIL" \ + '{calls: $calls, status: $status, reasons: $reasons, sites: $sites, + markers: $markers, detail: $detail}' + ) +} + +DISPATCH="$(exercise_leg_dispatch host)" +matrix_check "a host build is probed before it is patched" "probe,host" \ + "$(printf '%s' "$DISPATCH" | jq -r '.calls')" +matrix_check "a host result retains the compaction-marker check" "OK" \ + "$(printf '%s' "$DISPATCH" | jq -r '.sites["compact-prompt-markers"]')" +matrix_check "a host result keeps probe and execution patch-site verdicts distinct" "OK OK" \ + "$(printf '%s' "$DISPATCH" | jq -r '.sites["probe:PATCH 10: child network environment"] + " " + .sites["PATCH 10: child network environment"]')" +matrix_check "a host result retains probe details beside execution details" "elf 2.1.261" \ + "$(printf '%s' "$DISPATCH" | jq -r '.detail.probe.format + " " + .detail.patchedVersion')" + +DISPATCH="$(exercise_leg_dispatch container)" +matrix_check "a container build is probed before it is patched" "probe,container" \ + "$(printf '%s' "$DISPATCH" | jq -r '.calls')" +matrix_check "a container result retains the compaction-marker check" "OK" \ + "$(printf '%s' "$DISPATCH" | jq -r '.sites["compact-prompt-markers"]')" + +DISPATCH="$(exercise_leg_dispatch probe)" +matrix_check "a probe-only build runs exactly one probe" "probe" \ + "$(printf '%s' "$DISPATCH" | jq -r '.calls')" + +DISPATCH="$(exercise_leg_dispatch host fail pass)" +matrix_check "a marker failure survives a successful host patch" "fail" \ + "$(printf '%s' "$DISPATCH" | jq -r '.status')" +matrix_check "a failed probe site is not overwritten by a successful host site" "FAIL OK" \ + "$(printf '%s' "$DISPATCH" | jq -r '.sites["probe:PATCH 10: child network environment"] + " " + .sites["PATCH 10: child network environment"]')" +case "$(printf '%s' "$DISPATCH" | jq -r '.reasons')" in + *"compact prompt marker(s) missing: end"*) + pass=$((pass + 1)); printf 'ok %s\n' "a marker failure keeps its missing-marker reason" ;; + *) + fail=$((fail + 1)); printf 'FAIL a marker failure lost its reason — got: %s\n' "$DISPATCH" ;; +esac + +DISPATCH="$(exercise_leg_dispatch host error pass)" +matrix_check "a probe error survives a successful host patch" "error" \ + "$(printf '%s' "$DISPATCH" | jq -r '.status')" +case "$(printf '%s' "$DISPATCH" | jq -r '.reasons')" in + *"CANARY INFRASTRUCTURE: the probe produced no usable result"*) + pass=$((pass + 1)); printf 'ok %s\n' "a merged probe error identifies itself as infrastructure" ;; + *) + fail=$((fail + 1)); printf 'FAIL a merged probe error was not tagged — got: %s\n' "$DISPATCH" ;; +esac + +DISPATCH="$(exercise_leg_dispatch host error fail)" +matrix_check "an execution failure outranks a probe error" "fail" \ + "$(printf '%s' "$DISPATCH" | jq -r '.status')" +printf '%s\n' "$DISPATCH" | jq -c \ + '. + {platform:"test-platform",mode:"host",downgraded:false,binary:"",tarball:""}' > "$MATRIX" +matrix_check "a probe error is excluded from grouped release evidence" \ + "- host patch failed [test-platform]" "$(matrix_reason_groups)" +matrix_check "a probe error retained in a failed row reaches infrastructure notes" \ + "- the probe produced no usable result" "$(matrix_error_notes)" + +# Marker drift breaks OpenAI compaction, not binary patching. Human alerts must preserve that +# distinction, including when an infrastructure failure from the execution tier shares the row. +MARKER_REASON="Compaction-prompt drift is not a patch failure: no patch site is broken" +MATRIX_SITES='{"compact-prompt-markers":"FAIL","published-content":"OK"}' \ + write_matrix "darwin-arm64:host:fail:$MARKER_REASON" "win32-x64:probe:fail:$MARKER_REASON" +EXTRA_REASONS="" +if only_compact_prompt_drift_failed; then + pass=$((pass + 1)); printf 'ok %s\n' "marker-only failures get their own human alert category" +else + fail=$((fail + 1)); printf 'FAIL %s\n' "marker-only failures get their own human alert category" +fi +EXTRA_REASONS="- on win32-x64, PATCH 5 changed from OK to SKIP +" +if only_compact_prompt_drift_failed; then + fail=$((fail + 1)); printf 'FAIL %s\n' "a baseline regression vetoes the marker-only human alert" +else + pass=$((pass + 1)); printf 'ok %s\n' "a baseline regression vetoes the marker-only human alert" +fi +EXTRA_REASONS="" +case "$(matrix_coverage_brief)" in + *"compaction prompt marker check failed"*"native execution not checked"*) + pass=$((pass + 1)); printf 'ok %s\n' "a failed probe row names the prompt marker check" ;; + *) + fail=$((fail + 1)); printf 'FAIL marker failure coverage label — got: %s\n' "$(matrix_coverage_brief)" ;; +esac +matrix_check "marker drift headline does not call it a patch break" \ + ":warning: *Claude Code 2.1.999 changed its compaction prompt* on 2 of 8 builds" \ + "$(compact_prompt_drift_headline 2.1.999 2 8)" +case "$(compact_prompt_drift_impact)" in + *"does not mean \`clodex patch\` is broken"*"no patch site failed"*'Prompt is too long'*"updated prompt markers"*) + pass=$((pass + 1)); printf 'ok %s\n' "marker drift impact explains the real user risk without prescribing rollback" ;; + *) + fail=$((fail + 1)); printf 'FAIL marker drift impact — got: %s\n' "$(compact_prompt_drift_impact)" ;; +esac + +MATRIX_SITES='{"compact-prompt-markers":"FAIL","PATCH 1: Agent tool model enum":"FAIL"}' \ + write_matrix "darwin-arm64:host:fail:$MARKER_REASON" +if matrix_failures_are_compact_prompt_drift_only; then + fail=$((fail + 1)); printf 'FAIL %s\n' "a patch-site failure cannot use the marker-only alert" +else + pass=$((pass + 1)); printf 'ok %s\n' "a patch-site failure cannot use the marker-only alert" +fi + +jq -n -c --arg r "- $MARKER_REASON +- CANARY INFRASTRUCTURE: host execution unavailable +" \ + '{platform:"darwin-arm64",mode:"host",status:"fail",reasons:$r,binary:"",tarball:"", + downgraded:false,sites:{"compact-prompt-markers":"FAIL"},markers:[],detail:{}}' > "$MATRIX" +if matrix_failures_are_compact_prompt_drift_only; then + pass=$((pass + 1)); printf 'ok %s\n' "infrastructure evidence does not recategorize marker drift as a patch break" +else + fail=$((fail + 1)); printf 'FAIL %s\n' "infrastructure evidence does not recategorize marker drift as a patch break" +fi + +# Render the real agent-facing prompt with stdin closed. The jq file argument must stay in the +# substitution: when it was split onto the next shell line, jq read stdin and rendered an empty +# verification loop under launchd (or hung forever on an interactive --retriage). +MATRIX_SITES='{"compact-prompt-markers":"FAIL","published-content":"OK"}' \ + write_matrix "darwin-arm64:host:fail:$MARKER_REASON" "win32-x64:probe:fail:$MARKER_REASON" +VERSION=2.1.999 +MAIN_SHA=1234567890abcdef +MAIN_SUBJECT="test canary" +BASE_VERSION=2.1.998 +WORK="$TMP/triage-work" +REPO_DIR="$TMP/triage-repo" +TRIAGE_PROMPT="$(investigation_prompt "$VERSION" "- $MARKER_REASON" "$TMP/run.log" /dev/null)" +case "$TRIAGE_PROMPT" in + *"for p in darwin-arm64 win32-x64;"*) + pass=$((pass + 1)); printf 'ok %s\n' "the generated verification loop includes every downloaded matrix row" ;; + *) + fail=$((fail + 1)); printf 'FAIL generated verification loop — got:\n%s\n' "$TRIAGE_PROMPT" ;; +esac +case "$TRIAGE_PROMPT" in + *'A `compact-prompt-markers` failure is NOT a patch failure'*"bundle-reader"*) + pass=$((pass + 1)); printf 'ok %s\n' "generated triage distinguishes marker drift from a patch failure" ;; + *) + fail=$((fail + 1)); printf 'FAIL generated marker triage branch is missing\n' ;; +esac + +# A container executes and mutates its install after the universal probe. Its generated MODE=probe +# reproduction must therefore prefer clodex's pristine backup, not re-probe the patched survivor. +REPRO_OUTPUT="$( + ( + WORK="$TMP/repro-marker" + VERSION=2.1.261 + REPO_DIR="$WORK/repo" + MATRIX="$WORK/matrix.jsonl" + mkdir -p "$WORK/clodex-home" "$WORK/linux-arm64/install" \ + "$WORK/linux-arm64/tweakcc" "$REPO_DIR/scripts" "$REPO_DIR/node_modules/tweakcc" + printf '{}\n' > "$WORK/clodex-home/config.json" + printf 'patched\n' > "$WORK/linux-arm64/install/claude" + printf 'pristine\n' > "$WORK/linux-arm64/tweakcc/native-binary.backup" + jq -n -c \ + '{platform:"linux-arm64",mode:"container",status:"fail",reasons:"marker missing", + binary:"",tarball:"",downgraded:false,sites:{"compact-prompt-markers":"FAIL"}, + markers:[],detail:{image:"node:24-bookworm",dockerPlatform:"linux/arm64"}}' > "$MATRIX" + cat > "$REPO_DIR/scripts/probe-patch-mechanism.mjs" <<'PROBE' +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +const binary = process.argv[2]; +const sandbox = path.dirname(path.dirname(binary)); +const expected = { + HOME: path.join(sandbox, 'home'), + CLODEX_HOME: path.join(sandbox, 'clodex-home'), + TWEAKCC_CONFIG_DIR: path.join(sandbox, 'tweakcc'), + TWEAKCC_CC_INSTALLATION_PATH: binary, +}; +const isolated = Object.entries(expected).every(([key, value]) => process.env[key] === value) + && ['CLODEX_TRACE', 'FORCE_COLOR', 'CLICOLOR_FORCE'].every(key => !(key in process.env)); +console.log(`probe-input=${readFileSync(binary, 'utf8').trim()}`); +console.log(`probe-environment=${isolated ? 'isolated' : 'leaked'}`); +PROBE + write_repro_script >/dev/null + CLODEX_TRACE=dirty FORCE_COLOR=1 CLICOLOR_FORCE=1 \ + MODE=probe "$WORK/repro.sh" linux-arm64 "$REPO_DIR" + ) +)" +case "$REPRO_OUTPUT" in + *"probe-input=pristine"*) + pass=$((pass + 1)); printf 'ok %s\n' "a container probe reproduction starts from its pristine backup" ;; + *) + fail=$((fail + 1)); printf 'FAIL container probe reproduction used the wrong bytes — got: %s\n' "$REPRO_OUTPUT" ;; +esac +case "$REPRO_OUTPUT" in + *"probe-environment=isolated"*) + pass=$((pass + 1)); printf 'ok %s\n' "a probe reproduction fences home, config and inherited diagnostics" ;; + *) + fail=$((fail + 1)); printf 'FAIL probe reproduction leaked its environment — got: %s\n' "$REPRO_OUTPUT" ;; +esac + +# 19d. The whole incident, end to end at the matrix level: 2.1.238 lost PATCH 5 on the two arm64 # Linux builds AND on win32-arm64. All three must fail, as one grouped cause, and the run # must not earn the green tick. write_matrix \ diff --git a/canary/clodex-patch-canary.sh b/canary/clodex-patch-canary.sh index c0b207c3..1b842ff4 100755 --- a/canary/clodex-patch-canary.sh +++ b/canary/clodex-patch-canary.sh @@ -588,14 +588,15 @@ write_repro_script() { printf '# TWEAKCC_CONFIG_DIR and TWEAKCC_CC_INSTALLATION_PATH, so it cannot touch the installed\n' printf '# Claude Code, ~/.clodex or ~/.tweakcc.\n' printf '#\n' - printf '# %s/repro.sh # this Mac, clodex origin/main\n' "$WORK" - printf '# %s/repro.sh linux-arm64 # that leg, exactly as the canary ran it\n' "$WORK" - printf '# %s/repro.sh linux-arm64 # the same leg against your fix\n' "$WORK" + printf '# %s/repro.sh # this Mac execution tier, origin/main\n' "$WORK" + printf '# %s/repro.sh linux-arm64 # strongest execution tier for that platform\n' "$WORK" + printf '# MODE=probe %s/repro.sh linux-arm64 # the universal bundle/marker probe tier\n' "$WORK" + printf '# %s/repro.sh linux-arm64 # the execution tier against your fix\n' "$WORK" printf '#\n' - printf '# The leg matches the platform: the host runs the real `clodex patch`; linux-arm64 and\n' - printf '# linux-arm64-musl run it inside a container; the rest run the bundle-patch probe,\n' - printf '# scripts/probe-patch-mechanism.mjs. Your worktree does NOT need to be built first —\n' - printf '# the host leg builds it if dist is missing and the container legs always rebuild.\n' + printf '# The canary attempted the probe for every downloaded build. The default reruns its execution tier:\n' + printf '# the host runs the real `clodex patch`; linux-arm64 and linux-arm64-musl run it inside a\n' + printf '# container; other platforms already default to the probe. Your worktree does NOT need to be\n' + printf '# built first — the host builds it if dist is missing and containers always rebuild.\n' printf '#\n' printf 'set -Eeuo pipefail\n' printf 'CANARY_WORK=%q\n' "$WORK" @@ -613,8 +614,8 @@ ENTRY="$(jq -s -c --arg p "$PLATFORM" 'map(select(.platform == $p)) | last // em jq -s -r 'map(" \(.platform) [\(.mode)] \(.status)")[]' "$CANARY_WORK/matrix.jsonl" >&2 exit 1 } -# The leg this platform actually got. MODE=container overrides it: if the canary ran without -# Docker, the recorded leg is a probe, and a probe never starts the binary it patched. +# The leg this platform actually got. MODE may select probe or container explicitly; for example, +# MODE=container adds execution when Docker was unavailable to the scheduled run. MODE="${MODE:-$(printf '%s' "$ENTRY" | jq -r '.mode')}" TARBALL="$(printf '%s' "$ENTRY" | jq -r '.tarball')" MEMBER=claude @@ -628,10 +629,12 @@ done SB="$(mktemp -d "$CANARY_WORK/repro.XXXXXX")" mkdir -p "$SB/install" "$SB/clodex-home" "$SB/tweakcc" "$SB/home" "$SB/out" -# The canary keeps a failing leg's binary but deletes a passing one's, and the host leg's pristine -# backup is the most trustworthy source of all — it is the bytes clodex itself snapshotted. +# The canary keeps a failing leg's binary but deletes a passing one's. Prefer a pristine backup +# from either real patch tier — those are the release bytes clodex itself snapshotted. if [ -f "$CANARY_WORK/tweakcc/native-binary.backup" ] && [ "$PLATFORM" = "$CANARY_HOST_PLATFORM" ]; then cp "$CANARY_WORK/tweakcc/native-binary.backup" "$SB/install/$MEMBER" +elif [ -f "$CANARY_WORK/$PLATFORM/tweakcc/native-binary.backup" ]; then + cp "$CANARY_WORK/$PLATFORM/tweakcc/native-binary.backup" "$SB/install/$MEMBER" elif [ -f "$CANARY_WORK/$PLATFORM/install/$MEMBER" ]; then cp "$CANARY_WORK/$PLATFORM/install/$MEMBER" "$SB/install/$MEMBER" elif [ -f "$CANARY_WORK/$PLATFORM/$MEMBER" ]; then @@ -685,8 +688,13 @@ case "$MODE" in # the probe runs from source. [ -d "$REPO/node_modules/tweakcc" ] || ( cd "$REPO" && corepack pnpm install --frozen-lockfile ) cd "$REPO" - exec node "$REPO/scripts/probe-patch-mechanism.mjs" "$SB/install/$MEMBER" \ - --label "$PLATFORM" --expect-version "$CANARY_VERSION" --scratch "$SB/probe" + exec env -u CLODEX_TRACE -u FORCE_COLOR -u CLICOLOR_FORCE \ + HOME="$SB/home" \ + CLODEX_HOME="$SB/clodex-home" \ + TWEAKCC_CONFIG_DIR="$SB/tweakcc" \ + TWEAKCC_CC_INSTALLATION_PATH="$SB/install/$MEMBER" \ + node "$REPO/scripts/probe-patch-mechanism.mjs" "$SB/install/$MEMBER" \ + --label "$PLATFORM" --expect-version "$CANARY_VERSION" --scratch "$SB/probe" ;; container) IMAGE="$(printf '%s' "$ENTRY" | jq -r '.detail.image // ""')" @@ -728,35 +736,42 @@ REPRO_BODY investigation_prompt() { local version="$1" reasons="$2" logfile="$3" cat < # exactly what the canary ran for that platform - $WORK/repro.sh # the same leg, against your fix -It picks the right leg for the platform, seeds a fresh sandbox, and redirects HOME, CLODEX_HOME, -TWEAKCC_CONFIG_DIR and TWEAKCC_CC_INSTALLATION_PATH for you. With no arguments it runs this Mac's -leg against origin/main. Your worktree does not need to be built first. Read it before you run it; -if you need a variant, copy it and edit the copy. +REPRODUCE — USE THE TIER THAT REPORTED THE FAILURE + MODE=probe $WORK/repro.sh # universal bundle/marker probe + $WORK/repro.sh # strongest recorded execution tier + $WORK/repro.sh # that execution tier against your fix +The default command picks the matrix mode; on host/container platforms it does not repeat the probe +that ran before that tier. Each command seeds a fresh sandbox and redirects HOME, CLODEX_HOME, +TWEAKCC_CONFIG_DIR and TWEAKCC_CC_INSTALLATION_PATH. With no arguments it runs this Mac's execution +tier against origin/main. Your worktree does not need to be built first. Read the script before you +run it; if you need a variant, copy it and edit the copy. Fix the ROOT CAUSE once. Several platforms failing together is almost always one defect — the last time this happened, all four ELF builds broke on a single line in the restore sweep. Do not -open four PRs, and do not fix one platform in a way that only works for that platform. Verify the -fix by re-running the repro for EVERY platform in the matrix above, failing and passing alike: -a change to the binary handling can break a format that was fine. +open four PRs, and do not fix one platform in a way that only works for that platform. Verify a +probe or prompt-marker fix with \`MODE=probe\` for every downloaded row in the matrix (mode other +than \`none\`), failing and passing alike. Also rerun each platform's default tier when binary +handling or real patch execution changed; +a format that was fine can still regress. HARD CONSTRAINTS - **Never run a bare \`clodex patch\` or \`node dist/cli.js patch\`.** With no @@ -819,9 +838,10 @@ WHAT TO DO 3. Fix it, following CLAUDE.md and .claude/skills/pr-verification/SKILL.md in full — including bumping PATCH_TRANSFORMS_VERSION if the transform set changed materially, and a test that fails before the fix and passes after. -4. Verify: \`pnpm typecheck && pnpm test && pnpm build\`, then re-run every platform in the matrix — - \`for p in $(jq -s -r 'map(.platform) | join(" ")' "$MATRIX"); do $WORK/repro.sh \$p ; done\` — - and confirm each one comes out clean on a pristine $version copy. +4. Verify: \`pnpm typecheck && pnpm test && pnpm build\`, then re-run every downloaded row in the + matrix — \`for p in $(jq -s -r 'map(select(.mode != "none") | .platform) | join(" ")' "$MATRIX"); + do $WORK/repro.sh \$p ; done\` — and confirm each one comes out clean on a pristine + $version copy. 5. Run an adversarial review panel over your own change before pushing, per the pr-verification skill. Fix what it finds; re-review after changes. 6. Open a PR against main with \`gh pr create\`. Write the summary line for a user who has never read @@ -838,9 +858,24 @@ Do not finish without sending the concluding message. EOF } +only_compact_prompt_drift_failed() { + [ -z "${EXTRA_REASONS:-}" ] && matrix_failures_are_compact_prompt_drift_only +} + +compact_prompt_drift_headline() { # compact_prompt_drift_headline + printf ':warning: *Claude Code %s changed its compaction prompt* on %s of %s builds' "$1" "$2" "$3" +} + +compact_prompt_drift_impact() { + printf '%s' '*This does not mean `clodex patch` is broken* — no patch site failed, and this finding ' + printf '%s' 'does not call for a patch rollback. Claude Code reworded its plain-text compaction ' + printf '%s' "request, so clodex's guard no longer forces text-only responses on OpenAI models and " + printf '%s' 'long sessions can die with "Prompt is too long". clodex needs updated prompt markers.' +} + launch_investigation() { local version="$1" reasons="$2" logfile="$3" - local session_name prompt out session_id + local session_name prompt out session_id finding session_name="clodex-patch-canary $version $(date +%Y%m%d%H%M)" SESSION_NAME="$session_name" SESSION_ID="" @@ -855,7 +890,11 @@ launch_investigation() { # starting a second one alongside a session that is still tracked. if [ "$(state_read | jq -r '.inflight // "null"')" != "null" ]; then log "WARN: an investigation is already tracked — not launching a second one" - slack ":warning: clodex patch canary found that Claude Code $version breaks \`clodex patch\`, but an investigation of $(state_read | jq -r '.inflight.version') is already running, so no second session was started. Details: $logfile" || true + finding="breaks \`clodex patch\`" + if [ "${COMPACT_PROMPT_DRIFT_ONLY:-0}" -eq 1 ]; then + finding="changed the compaction prompt clodex recognizes" + fi + slack ":warning: clodex patch canary found that Claude Code $version $finding, but an investigation of $(state_read | jq -r '.inflight.version') is already running, so no second session was started. Details: $logfile" || true return 1 fi if [ ! -x "$CLAUDE_BIN" ]; then @@ -1183,9 +1222,9 @@ if [ "$BASE_PLATFORMS" != "{}" ]; then add_soft "your favourites/aliases changed since the $BASE_VERSION baseline, so $PLATFORM's checks were not compared; its baseline is re-taken from $VERSION" continue fi - # A leg's checks are named per mode — patch sites for host/container, probe check names for a - # probe. Comparing across modes reports every name in the other set as a lost site, so a single - # Docker outage would produce a page of invented regressions on the run after it. + # A host/container result now carries the probe checks plus the real command's patch report; a + # probe-only result carries no execution report. Comparing across modes still reports names from + # the extra tier as lost, so a single Docker outage would invent regressions on the next run. # An entry recorded before this field existed carries no mode, so its names cannot be placed in # either set. Comparing it anyway does the exact damage the mode check exists to prevent — it # reported all 16 probe names as dropped on two consecutive container runs — so an entry with no @@ -1264,6 +1303,8 @@ REASONS="$EXTRA_REASONS$(matrix_reason_groups)" [ -n "$(matrix_reason_groups)" ] && REASONS="$REASONS " [ -n "$FAILED_PLATFORMS" ] || [ -n "$EXTRA_REASONS" ] || REASONS="" +COMPACT_PROMPT_DRIFT_ONLY=0 +only_compact_prompt_drift_failed && COMPACT_PROMPT_DRIFT_ONLY=1 ERROR_COUNT="$(matrix_count error)" DOWNGRADED="$(jq -s -r 'map(select(.downgraded == true) | .platform) | join(", ")' "$MATRIX")" @@ -1319,9 +1360,9 @@ if [ -z "$REASONS" ]; then else UPDATE_LINE="You are on $INSTALLED_VERSION, so the update is yours to take whenever you like. Re-run \`clodex patch\` afterwards to re-apply your aliases." fi - # What was actually established, in the words of what was actually run. A probe leg exercises no - # patch site and never starts the binary, so it may not be described as "clodex patch applies - # cleanly" — that phrase is only true of the host and container legs. + # What was actually established, in the words of what was actually run. The probe applies every + # patch transform to extracted JavaScript but never runs the real `clodex patch` command or starts + # its result, so "clodex patch applies cleanly" is still true only of host and container legs. FULL_LEGS="$(jq -s -r 'map(select(.status == "pass" and (.mode == "host" or .mode == "container")) | .platform) | join(", ")' "$MATRIX")" FULL_COUNT="$(jq -s -r 'map(select(.status == "pass" and (.mode == "host" or .mode == "container"))) | length' "$MATRIX")" PROBE_COUNT="$(jq -s -r 'map(select(.status == "pass" and .mode == "probe")) | length' "$MATRIX")" @@ -1357,7 +1398,11 @@ The real \`clodex patch\` ran on $FULL_COUNT of $TOTAL_COUNT builds ($FULL_LEGS) Tested against clodex origin/main \`${MAIN_SHA:0:8}\` (not the published $(node -p "require('$REPO_DIR/package.json').version" 2>/dev/null || echo release)), so this is a verdict on main. $UPDATE_LINE$( [ -n "$SOFT_NOTES" ] && printf '\n\n:information_source: Worth knowing:\n%s' "$SOFT_NOTES" )$( [ "$COVERAGE_COMPLETE" -eq 1 ] || printf '\n\n%s' "$COVERAGE" )" || true else - log "FAIL: Claude Code $VERSION did not patch cleanly:" + if [ "$COMPACT_PROMPT_DRIFT_ONLY" -eq 1 ]; then + log "FAIL: Claude Code $VERSION changed the compaction prompt clodex recognizes:" + else + log "FAIL: Claude Code $VERSION did not patch cleanly:" + fi printf '%s' "$REASONS" KEEP_WORK=1 # the investigation needs the sandbox write_repro_script @@ -1407,7 +1452,10 @@ else LAST_GOOD="" [ -z "$LAST_COMPLETE" ] || LAST_GOOD=" Last release that came out clean on every build: *$LAST_COMPLETE*." - if [ "$HOST_STATUS" = "fail" ] && [ "$INSTALLED_VERSION" = "$VERSION" ]; then + if [ "$COMPACT_PROMPT_DRIFT_ONLY" -eq 1 ]; then + HEADLINE="$(compact_prompt_drift_headline "$VERSION" "$(matrix_count fail)" "$TOTAL_COUNT")" + IMPACT="$(compact_prompt_drift_impact)$LAST_GOOD" + elif [ "$HOST_STATUS" = "fail" ] && [ "$INSTALLED_VERSION" = "$VERSION" ]; then IMPACT="*You are already ON $VERSION and it cannot be patched* — your aliases and effort settings are off until this is fixed. Roll back, or wait for the fix.$LAST_GOOD" elif [ "$HOST_STATUS" = "fail" ]; then IMPACT="*Do not update Claude Code yet* — you are on $INSTALLED_VERSION and patching $VERSION fails on this Mac. Your current patched install keeps working; the risk is only at update time.$LAST_GOOD" diff --git a/scripts/probe-patch-mechanism.mjs b/scripts/probe-patch-mechanism.mjs index e67ebeec..7781f54e 100644 --- a/scripts/probe-patch-mechanism.mjs +++ b/scripts/probe-patch-mechanism.mjs @@ -71,6 +71,7 @@ import { restoreBunCompiledPointer, shimBunCompiledPointer, } from '../src/bun-compiled-pointer.ts'; +import { checkClaudeCodeCompactPromptMarkers } from '../src/claude-code-compact-prompt.ts'; // `src/` spells its own imports the TypeScript way — `./model-aliases.js` for a file that is // really `./model-aliases.ts`. tsup and vitest both understand that; bare `node` does not, and @@ -363,6 +364,26 @@ try { info.bundleModules = bundle ? bundle.modules.length : null; info.readMs = Date.now() - started; + // The runtime text-only guard intentionally keys on Claude Code's exact prompt. Check those same + // bytes in each platform's own extracted bundle so upstream wording drift fails the release + // canary instead of silently disabling the guard in production. + const compactPromptMarkers = checkClaudeCodeCompactPromptMarkers(source); + info.compactPromptMarkers = { missing: compactPromptMarkers.missing }; + record( + 'compact-prompt-markers', + compactPromptMarkers.ok, + compactPromptMarkers.detail, + compactPromptMarkers.ok ? undefined : [ + 'Compaction-prompt drift is not a patch failure: patching itself is fine for this finding, ' + + 'and no patch site is broken. On a complete bundle extraction, Claude Code reworded its ' + + "compaction prompt, which stops clodex's text-only guard from firing; auto-compaction " + + 'then fails on OpenAI models and long sessions die with "Prompt is too long". First ' + + 'confirm the extracted source still contains both compaction builders, because a ' + + 'bundle-reader omission has the same symptom. Then update the markers in ' + + 'src/claude-code-compact-prompt.ts to the new wording from this bundle.', + ], + ); + if (readShim) restoreEntryModuleName(scratch, readShim, { resign: false }); record( 'seed-round-trip', diff --git a/src/claude-code-compact-prompt.ts b/src/claude-code-compact-prompt.ts new file mode 100644 index 00000000..2e558301 --- /dev/null +++ b/src/claude-code-compact-prompt.ts @@ -0,0 +1,38 @@ +/** + * Verbatim Claude Code compaction-prompt text that clodex depends on. + * + * The runtime guard and the per-build release probe both import this object so + * a clodex marker update cannot leave either one checking a stale private copy. + */ +export const CLAUDE_CODE_COMPACT_PROMPT_MARKERS = Object.freeze({ + start: 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.', + end: 'REMINDER: Do NOT call any tools. Respond with plain text only', +} as const); + +export type ClaudeCodeCompactPromptMarkerName = + keyof typeof CLAUDE_CODE_COMPACT_PROMPT_MARKERS; + +export interface ClaudeCodeCompactPromptMarkerCheck { + ok: boolean; + missing: ClaudeCodeCompactPromptMarkerName[]; + detail: string; +} + +/** Report which strict compaction-prompt markers are absent from a bundle. */ +export function checkClaudeCodeCompactPromptMarkers( + source: string, +): ClaudeCodeCompactPromptMarkerCheck { + const entries = Object.entries(CLAUDE_CODE_COMPACT_PROMPT_MARKERS) as Array< + [ClaudeCodeCompactPromptMarkerName, string] + >; + const missing = entries + .filter(([, marker]) => !source.includes(marker)) + .map(([name]) => name); + return { + ok: missing.length === 0, + missing, + detail: missing.length === 0 + ? 'both strict compaction prompt markers are present' + : `missing strict compaction prompt marker(s): ${missing.join(', ')}`, + }; +} diff --git a/src/proxy.ts b/src/proxy.ts index 232ddda9..174b0d82 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -619,6 +619,7 @@ export async function startProxyCatalog( openAiOAuth, claudeSessionId, maxTools: maxToolsForNpm(route.npm), + log: plog, reasoningMetadata: { providerId: route.providerId, apiBaseUrl: route.baseURL, diff --git a/src/sdk-adapter.ts b/src/sdk-adapter.ts index c7f6ac1a..47314464 100644 --- a/src/sdk-adapter.ts +++ b/src/sdk-adapter.ts @@ -23,6 +23,7 @@ import type { AnthropicRequestMessage, AnthropicToolDefinition } from './proxy-t import { anthropicErrorType, upstreamHttpStatus } from './upstream-error.js'; import { upstreamMaxRetries } from './upstream-retry.js'; import { emitParentNotice } from './parent-notice.js'; +import { CLAUDE_CODE_COMPACT_PROMPT_MARKERS } from './claude-code-compact-prompt.js'; import { CLAUDE_CODE_BILLING_HEADER_PREFIX } from './oauth/claude-identity.js'; export { silenceSdkWarnings }; @@ -90,6 +91,8 @@ export interface TranslateRequestOptions { claudeSessionId?: string; /** Hard cap on tools sent to the provider (e.g. Groq: 128). Excess tools are silently dropped. */ maxTools?: number; + /** Immediate trace-log sink; diagnostics must call it before terminal-warning suppression. */ + log?: (message: string) => void; } const CLAUDE_SESSION_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; @@ -467,8 +470,10 @@ export function translateToolChoice(tc: AnthropicRequest['tool_choice']): SdkCal return undefined; } -const COMPACT_TEXT_ONLY_START = 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.'; -const COMPACT_TEXT_ONLY_END = 'REMINDER: Do NOT call any tools. Respond with plain text only'; +const { + start: COMPACT_TEXT_ONLY_START, + end: COMPACT_TEXT_ONLY_END, +} = CLAUDE_CODE_COMPACT_PROMPT_MARKERS; /** * Claude Code forks its reactive-compaction turn with the SAME tool definitions @@ -512,6 +517,83 @@ function isClaudeCodeCompactRequest(body: AnthropicRequest): boolean { text.startsWith(COMPACT_TEXT_ONLY_START) && text.includes(COMPACT_TEXT_ONLY_END)); } +// This sentence is the reason the compaction fork needs a text-only response, +// not decoration around it. It occurs in both prompt builders across 27 extracted +// 2.1.238–2.1.260 bundles; all eight platforms are represented for 2.1.257 and +// 2.1.260. The recognizer deliberately covers only rewordings that preserve this +// anchor and a bounded opening grammar. The imperative must start at byte zero +// (after an optional known severity label), so quoted copies with a preamble stay +// out; a raw copy pasted with no preamble is indistinguishable from Claude Code's +// own prompt. The START guard excludes known partial envelopes. A +// START-intact/END-reworded prompt also stays invisible by design because it is +// indistinguishable from a user quoting the header; the per-build release probe +// catches removal or in-place rewording of either current marker. +const COMPACT_DRIFT_ANCHOR = 'Tool calls will be REJECTED and will waste your only turn'; +const COMPACT_DRIFT_ANCHOR_LINE = new RegExp( + `(?:^|\\n)[-*•\\s]{0,4}${COMPACT_DRIFT_ANCHOR}`, +); +const COMPACT_DRIFT_OPENING = /^(?:(?:critical|important|warning|caution|urgent|notice):\s*)?(?:respond|return|answer|output|write|provide)\b(?=[^\n]{1,160}(?:\n|$))(?=[^\n]*\b(?:text\s+only|plain\s+text\s+only|only\s+(?:plain\s+)?text)\b)(?=[^\n]*\b(?:do not|don't|never|without)\b[^\n]{0,48}\btools?\b)/i; + +function looksLikeDriftedClaudeCodeCompactRequest(body: AnthropicRequest): boolean { + if (body.diagnostics !== undefined) return false; + + const finalMessage = body.messages.at(-1); + if (!finalMessage || finalMessage.role !== 'user') return false; + const texts = typeof finalMessage.content === 'string' + ? [finalMessage.content] + : finalMessage.content + .filter(block => block.type === 'text') + .map(block => block.text ?? ''); + return texts.some(text => + !text.startsWith(COMPACT_TEXT_ONLY_START) + && COMPACT_DRIFT_OPENING.test(text) + && COMPACT_DRIFT_ANCHOR_LINE.test(text)); +} + +function claudeCodeVersionFromRequest(body: AnthropicRequest): string | undefined { + const texts = typeof body.system === 'string' + ? [body.system] + : (body.system ?? []).map(block => typeof block === 'string' ? block : block.text ?? ''); + for (const text of texts) { + if (!text.startsWith(CLAUDE_CODE_BILLING_HEADER_PREFIX)) continue; + const match = text.match(/\bcc_version=([0-9A-Za-z][0-9A-Za-z._+-]{0,63})(?:;|\s|$)/); + if (match) return match[1]; + } + return undefined; +} + +const warnedCompactPromptDrifts = new Set(); +const MAX_COMPACT_PROMPT_DRIFT_WARNINGS = 3; + +function reportClaudeCodeCompactPromptDrift( + body: AnthropicRequest, + log?: (message: string) => void, +): void { + if (!looksLikeDriftedClaudeCodeCompactRequest(body)) return; + const version = claudeCodeVersionFromRequest(body); + const signature = version ?? 'unknown-version'; + try { log?.(`possible Claude Code compact prompt drift: ${signature}`); } catch { /* ignore */ } + if (warnedCompactPromptDrifts.has(signature)) return; + if (warnedCompactPromptDrifts.size >= MAX_COMPACT_PROMPT_DRIFT_WARNINGS) return; + warnedCompactPromptDrifts.add(signature); + const versionText = version ? ` from Claude Code ${version}` : ''; + // emitParentNotice, not a bare process.stderr.write: while `clodex claude` has + // Claude Code running, launch.ts mutes the parent's stderr to protect the TUI. + emitParentNotice( + `clodex: warning: a request${versionText} looks like a compaction turn, but its prompt no longer ` + + "matches clodex's text-only guard. Tools were left enabled and compaction may fail. Please " + + 'report this at https://github.com/bman654/clodex/issues', + ); + if (warnedCompactPromptDrifts.size === MAX_COMPACT_PROMPT_DRIFT_WARNINGS) { + emitParentNotice('clodex: warning: further compact-prompt drift warnings suppressed.'); + } +} + +/** Test seam: the warning cap is process-wide and would leak between cases. */ +export function resetCompactPromptDriftWarningsForTests(): void { + warnedCompactPromptDrifts.clear(); +} + export function translateRequest( body: AnthropicRequest, npm: string, @@ -538,6 +620,7 @@ export function translateRequest( // definitions intact for prompt-cache prefix reuse; toolChoice='none' below // makes them unavailable at the provider API rather than by prompt compliance. const compactRequest = isClaudeCodeCompactRequest(body); + if (!compactRequest) reportClaudeCodeCompactPromptDrift(body, options?.log); let upstreamTools = resolveUpstreamTools( body.tools as unknown as AnthropicToolDefinition[] | undefined, messages as unknown as AnthropicRequestMessage[], diff --git a/src/server/router.ts b/src/server/router.ts index 4490582b..a29d1093 100644 --- a/src/server/router.ts +++ b/src/server/router.ts @@ -427,6 +427,7 @@ async function handleAnthropicMessages( upstreamModelId: upstreamModelId(model), }, maxTools: npmMaxTools, + log: plog, }); const clientWantsStream = Boolean(body.stream); // Use the display name in the response model field when masking is on — Claude diff --git a/tests/probe-patch-mechanism.test.ts b/tests/probe-patch-mechanism.test.ts new file mode 100644 index 00000000..68826205 --- /dev/null +++ b/tests/probe-patch-mechanism.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { buildFakeNativeClaude } from './bun-blob-fixture.js'; +import { CLAUDE_FIXTURE } from './fixtures/claude-bundle.js'; + +const REPO = dirname(dirname(fileURLToPath(import.meta.url))); + +describe('the full patch-mechanism probe', () => { + it('fails a fake bundle whose compact prompt is missing one strict marker', () => { + const dir = mkdtempSync(join(tmpdir(), 'clodex-probe-integration-')); + try { + const binary = join(dir, 'claude'); + const scratch = join(dir, 'scratch'); + const hook = join(dir, 'resolve-hook.mjs'); + const fakeTweakcc = join(dir, 'fake-tweakcc.mjs'); + const fixtureModule = pathToFileURL(join(REPO, 'tests/bun-blob-fixture.ts')).href; + + // Independent oracle copied from an extracted bundle: importing the production values would + // let one bad edit change both the implementation and the integration fixture. + const strictStart = 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.'; + const source = [CLAUDE_FIXTURE, strictStart, 'x'.repeat(1_000_000)].join('\n'); + writeFileSync(binary, buildFakeNativeClaude('test-version', [ + { name: '/$bunfs/root/claude', contents: source }, + ])); + + writeFileSync(fakeTweakcc, ` +import { readFileSync, writeFileSync } from 'node:fs'; +import { parseBunBlob, rebuildFakeNativeClaude } from ${JSON.stringify(fixtureModule)}; +export async function tryDetectInstallation({ path }) { + return { path, version: 'test-version', kind: 'native' }; +} +export async function readContent(installation) { + return parseBunBlob(readFileSync(installation.path)).contents[0]; +} +export async function writeContent(installation, content) { + const binary = readFileSync(installation.path); + writeFileSync( + installation.path, + rebuildFakeNativeClaude(binary, 'test-version', (index, previous) => + index === 0 ? content : previous), + ); +} +`); + writeFileSync(hook, ` +import { registerHooks } from 'node:module'; +const fakeTweakcc = new URL('./fake-tweakcc.mjs', import.meta.url).href; +registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier === 'tweakcc') return { shortCircuit: true, url: fakeTweakcc }; + return nextResolve(specifier, context); + }, +}); +`); + + const result = spawnSync(process.execPath, [ + '--import', hook, + join(REPO, 'scripts/probe-patch-mechanism.mjs'), + binary, + '--json', + '--scratch', scratch, + ], { + cwd: REPO, + encoding: 'utf8', + env: { + ...process.env, + TWEAKCC_CONFIG_DIR: join(dir, 'tweakcc-config'), + }, + }); + + expect(result.status).toBe(1); + expect(result.stderr).not.toMatch(/\n\s+at /); + const report: unknown = JSON.parse(result.stdout); + expect(report).toMatchObject({ + verdict: 'fail', + compactPromptMarkers: { missing: ['end'] }, + checks: expect.arrayContaining([ + expect.objectContaining({ + name: 'compact-prompt-markers', + ok: false, + detail: 'missing strict compaction prompt marker(s): end', + }), + ]), + }); + const serialized = JSON.stringify(report); + expect(serialized).toContain('Compaction-prompt drift is not a patch failure'); + expect(serialized).toContain('no patch site is broken'); + expect(serialized).toContain('Claude Code reworded its'); + expect(serialized).toContain('compaction prompt'); + expect(serialized).toContain('bundle-reader omission has the same symptom'); + expect(serialized).toContain("clodex's text-only guard from firing"); + expect(serialized).toContain('auto-compaction then fails on OpenAI models'); + expect(serialized).toContain(String.raw`long sessions die with \"Prompt is too long\"`); + expect(serialized).toContain('src/claude-code-compact-prompt.ts'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/probe-patch-sites.test.ts b/tests/probe-patch-sites.test.ts index c40fde7b..53c95ec0 100644 --- a/tests/probe-patch-sites.test.ts +++ b/tests/probe-patch-sites.test.ts @@ -1,8 +1,8 @@ // The patch-site half of the release canary's probe (scripts/probe-patch-sites.mjs). // -// The canary runs that probe against a real Claude Code build for every platform that has no -// container image — win32-x64, win32-arm64, linux-x64, linux-x64-musl, darwin-x64. It used to -// check executable-format handling only, so it reported win32-arm64 as a clean pass on Claude +// The scheduled canary targets all eight published platforms and runs this probe against each +// build it successfully downloads, before host and container builds receive execution checks. It +// used to check executable-format handling only, so it reported win32-arm64 as a clean pass on Claude // Code 2.1.238 while `PATCH 5: model picker options` no longer matched that build's bundle at // all. It now applies the real transforms, which means two new ways for it to be wrong: // @@ -21,13 +21,14 @@ import { } from '../scripts/probe-patch-sites.mjs'; import { applyClodexPatches } from '../src/patch-transforms.js'; import { CLAUDE_FIXTURE } from './fixtures/claude-bundle.js'; +import { checkClaudeCodeCompactPromptMarkers } from '../src/claude-code-compact-prompt.js'; describe('the canary probe against a healthy bundle', () => { it('exercises every patch site the transform set has, and no other', () => { const { sites, failures } = checkPatchSites(CLAUDE_FIXTURE); - // Exact list, in order. A site the probe does not know about is a site nothing vouches for on - // five of the eight published builds, and a name drifting is as invisible as a name removed. + // Exact list, in order. A site the probe does not know about is unchecked on the five + // probe-only platform builds, and a name drifting is as invisible as a name removed there. expect(sites.map((s) => s.name)).toEqual([...EXPECTED_PATCH_SITES]); expect(failures).toEqual([]); }); @@ -129,3 +130,38 @@ describe('the canary probe against a bundle whose anchors drifted', () => { expect(failures[2]).toContain("clodex's transform set"); }); }); + +describe('the canary probe against Claude Code compact prompt drift', () => { + it('names either missing production marker and fails when both have drifted', () => { + // Independent oracle copied from an extracted 2.1.260 bundle. Importing the production marker + // values here would let a bad edit change both the implementation and its expected fixture. + const start = 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.'; + const end = 'REMINDER: Do NOT call any tools. Respond with plain text only'; + const healthy = ['bundle prefix', start, 'summary instructions', end, 'bundle suffix'].join('\n'); + + expect(checkClaudeCodeCompactPromptMarkers(healthy)).toEqual({ + ok: true, + missing: [], + detail: 'both strict compaction prompt markers are present', + }); + expect(checkClaudeCodeCompactPromptMarkers( + healthy.replace(start, 'REWORDED COMPACT START'), + )).toEqual({ + ok: false, + missing: ['start'], + detail: 'missing strict compaction prompt marker(s): start', + }); + expect(checkClaudeCodeCompactPromptMarkers( + healthy.replace(end, 'REWORDED COMPACT END'), + )).toEqual({ + ok: false, + missing: ['end'], + detail: 'missing strict compaction prompt marker(s): end', + }); + expect(checkClaudeCodeCompactPromptMarkers('both prompt markers were reworded')).toEqual({ + ok: false, + missing: ['start', 'end'], + detail: 'missing strict compaction prompt marker(s): start, end', + }); + }); +}); diff --git a/tests/proxy.test.ts b/tests/proxy.test.ts index 47ed15d3..e1ebeed2 100644 --- a/tests/proxy.test.ts +++ b/tests/proxy.test.ts @@ -9,6 +9,8 @@ import { makeRouteResolver, resolveCatalogModelAliases } from '../src/catalog.js import { getProxyDebugLogPath } from '../src/trace-log.js'; import { anthropicMessagesEndpoint, estimateAnthropicInputTokens } from '../src/anthropic-endpoints.js'; import type { LocalProvider, ModelAlias } from '../src/types.js'; +import { resetCompactPromptDriftWarningsForTests } from '../src/sdk-adapter.js'; +import { installParentNoticeSink } from '../src/parent-notice.js'; /** POST JSON to a local proxy via node:http (avoids vi.stubGlobal('fetch') interception). */ function postToProxy( @@ -879,6 +881,58 @@ describe('translated request cancellation', () => { }, 20_000); }); +describe('compact-prompt drift trace logging', () => { + it('records every sighting on the default proxy translation route', async () => { + const dir = mkdtempSync(join(tmpdir(), 'clodex-compact-drift-proxy-')); + const debugLogPath = join(dir, 'debug.log'); + const notices: string[] = []; + const releaseNotices = installParentNoticeSink(line => notices.push(line)); + resetCompactPromptDriftWarningsForTests(); + const route: ProxyRoute = { + aliasId: 'clodex:test:translated-model', + realModelId: 'translated-model', + displayName: 'Translated Model', + upstreamUrl: '', + apiKey: 'provider-key', + modelFormat: 'openai', + npm: 'missing-sdk-provider-for-test', + providerId: 'test-provider', + }; + const handle = await startProxyCatalog( + [route], + route.aliasId, + true, + undefined, + debugLogPath, + ); + + try { + const response = await postToProxy(handle.port, handle.token, { + model: route.aliasId, + max_tokens: 100, + messages: [{ + role: 'user', + content: [ + 'Return only plain text. Never invoke any tools.', + '- Tool calls will be REJECTED and will waste your only turn — you will fail the task.', + ].join('\n'), + }], + stream: false, + }); + + expect(response.status).toBe(502); + expect(readFileSync(debugLogPath, 'utf8')) + .toContain('possible Claude Code compact prompt drift: unknown-version'); + expect(notices).toHaveLength(1); + } finally { + handle.close(); + resetCompactPromptDriftWarningsForTests(); + releaseNotices(); + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe('SDK translated error logging', () => { it('returns an HTTP error when request translation throws instead of leaving the client pending', async () => { const dir = mkdtempSync(join(tmpdir(), 'clodex-sdk-translation-error-')); diff --git a/tests/sdk-adapter.test.ts b/tests/sdk-adapter.test.ts index c512258a..95213419 100644 --- a/tests/sdk-adapter.test.ts +++ b/tests/sdk-adapter.test.ts @@ -15,6 +15,7 @@ import { claudeSessionPromptCacheKey, sdkTranslationErrorSignature, resetServiceTierWarningForTests, + resetCompactPromptDriftWarningsForTests, silenceSdkWarnings, } from '../src/sdk-adapter.js'; import { installParentNoticeSink } from '../src/parent-notice.js'; @@ -708,6 +709,354 @@ describe('translateRequest', () => { { name: 'Grep', input_schema: { type: 'object' } }, ]; + const driftedCompactPrompt = (opening = 'IMPORTANT: Return only plain text. Never invoke any tools.') => [ + opening, + '', + '- Do not use Read, Bash, or any other tool.', + '- You already have all the context you need in the conversation above.', + '- Tool calls will be REJECTED and will waste your only turn — you will fail the task.', + '- Put your summary in plain text.', + '', + 'Create a detailed summary of the conversation so far.', + '', + 'FINAL NOTE: Return plain text without invoking tools.', + ].join('\n'); + + it('warns once for a reworded compact prompt and logs every duplicate sighting', () => { + const notices: string[] = []; + const traces: string[] = []; + const releaseNotices = installParentNoticeSink(line => notices.push(line)); + resetCompactPromptDriftWarningsForTests(); + try { + const body = { + model: 'gpt-5.6-sol', + system: [{ + text: 'x-anthropic-billing-header: cc_version=2.1.261.a1b; cc_entrypoint=cli;', + }], + messages: [{ + role: 'user' as const, + content: driftedCompactPrompt('Respond with plain text only. Do not call any tools.'), + }], + tools: shellSessionTools, + tool_choice: { type: 'auto' as const }, + }; + + const first = translateRequest(body, '@ai-sdk/openai', { openAiOAuth: true, log: m => traces.push(m) }); + const duplicate = translateRequest(body, '@ai-sdk/openai', { openAiOAuth: true, log: m => traces.push(m) }); + + // The loose tier diagnoses only. The strict tier still owns behavior and + // deliberately leaves tools available when its exact markers are absent. + expect(first.toolChoice).toBe('auto'); + expect(duplicate.toolChoice).toBe('auto'); + expect(notices).toHaveLength(1); + expect(notices[0]).toContain('Claude Code 2.1.261.a1b'); + expect(notices[0]).toContain("no longer matches clodex's text-only guard"); + expect(notices[0]).toContain('https://github.com/bman654/clodex/issues'); + // Trace logging happens before terminal dedupe, preserving both sightings. + expect(traces).toEqual([ + 'possible Claude Code compact prompt drift: 2.1.261.a1b', + 'possible Claude Code compact prompt drift: 2.1.261.a1b', + ]); + } finally { + resetCompactPromptDriftWarningsForTests(); + releaseNotices(); + } + }); + + it('warns on a block-array compact turn when the header changes but its reminder remains', () => { + const notices: string[] = []; + const traces: string[] = []; + const releaseNotices = installParentNoticeSink(line => notices.push(line)); + resetCompactPromptDriftWarningsForTests(); + try { + const result = translateRequest({ + model: 'gpt-5.6-sol', + messages: [{ + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'call_1', content: 'shell output' }, + { type: 'text', text: `${driftedCompactPrompt()}${CC_COMPACT_TAIL}` }, + ], + }], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true, log: message => traces.push(message) }); + + expect(result.toolChoice).toBe('auto'); + expect(notices).toHaveLength(1); + expect(traces).toEqual(['possible Claude Code compact prompt drift: unknown-version']); + } finally { + resetCompactPromptDriftWarningsForTests(); + releaseNotices(); + } + }); + + it('caps compact-prompt drift notices while continuing to trace later sightings', () => { + const notices: string[] = []; + const traces: string[] = []; + const releaseNotices = installParentNoticeSink(line => notices.push(line)); + resetCompactPromptDriftWarningsForTests(); + try { + for (const version of ['2.1.261.0', '2.1.262.0', '2.1.263.0', '2.1.264.0']) { + translateRequest({ + model: 'gpt-5.6-sol', + system: `x-anthropic-billing-header: cc_version=${version}; cc_entrypoint=cli;`, + messages: [{ role: 'user', content: driftedCompactPrompt() }], + tools: shellSessionTools, + tool_choice: { type: 'auto' }, + }, '@ai-sdk/openai', { openAiOAuth: true, log: m => traces.push(m) }); + } + + expect(notices.filter(line => line.includes('looks like a compaction turn'))).toHaveLength(3); + expect(notices.at(-1)).toContain('further compact-prompt drift warnings suppressed'); + expect(notices).toHaveLength(4); + expect(traces).toHaveLength(4); + expect(traces.at(-1)).toBe('possible Claude Code compact prompt drift: 2.1.264.0'); + } finally { + resetCompactPromptDriftWarningsForTests(); + releaseNotices(); + } + }); + + it('uses only a bounded version from a real billing-header system block', () => { + const cases = [ + `quoted metadata: x-anthropic-billing-header: cc_version=2.1.999; cc_entrypoint=cli;`, + `x-anthropic-billing-header: cc_version=${'a'.repeat(65)}; cc_entrypoint=cli;`, + ]; + + for (const system of cases) { + const notices: string[] = []; + const traces: string[] = []; + const releaseNotices = installParentNoticeSink(line => notices.push(line)); + resetCompactPromptDriftWarningsForTests(); + try { + translateRequest({ + model: 'gpt-5.6-sol', + system, + messages: [{ role: 'user', content: driftedCompactPrompt() }], + tools: shellSessionTools, + }, '@ai-sdk/openai', { openAiOAuth: true, log: message => traces.push(message) }); + + expect(notices).toHaveLength(1); + expect(notices[0]).not.toContain('from Claude Code'); + expect(traces).toEqual(['possible Claude Code compact prompt drift: unknown-version']); + } finally { + resetCompactPromptDriftWarningsForTests(); + releaseNotices(); + } + } + }); + + it('does not warn on probes, quotes, tool results, split blocks, history, or prefills', () => { + const notices: string[] = []; + const traces: string[] = []; + const releaseNotices = installParentNoticeSink(line => notices.push(line)); + resetCompactPromptDriftWarningsForTests(); + const base = { + model: 'gpt-5.6-sol', + tools: shellSessionTools, + tool_choice: { type: 'auto' as const }, + }; + try { + const requests = [ + { + ...base, + diagnostics: { previous_message_id: null }, + messages: [{ role: 'user' as const, content: driftedCompactPrompt() }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: `Why does Claude Code send this?\n\n${driftedCompactPrompt()}\n\nExplain it.`, + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: `QUOTED PROMPT: ${driftedCompactPrompt('Return only plain text. Never invoke any tools.')}`, + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: driftedCompactPrompt('STOP: Respond with plain text only. Do not call any tools.'), + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + 'Return only plain text. Never invoke any tools.', + '- Tool calls will be REJECTED and nothing else.', + ].join('\n'), + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + 'Return only plain text. Never invoke any tools.', + `This prose quotes "${'- Tool calls will be REJECTED and will waste your only turn'}" for discussion.`, + ].join('\n'), + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + `Respond with plain text only. Never ${'delay '.repeat(12)}invoke tools.`, + '- Tool calls will be REJECTED and will waste your only turn — explain this.', + ].join('\n'), + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + `Respond with plain text only. Never invoke tools. ${'Continue briefly. '.repeat(10)}`, + '- Tool calls will be REJECTED and will waste your only turn — explain this.', + ].join('\n'), + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + 'Respond quickly, do not stop.', + '- Tool calls will be REJECTED and will waste your only turn — explain this.', + ].join('\n'), + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + 'Respond immediately. Do not call any tools.', + '- Tool calls will be REJECTED and will waste your only turn — explain this.', + ].join('\n'), + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + 'Respond with plain text only. Be concise.', + '- Tool calls will be REJECTED and will waste your only turn — explain this.', + ].join('\n'), + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + 'Respondent: use plain text only and do not call tools.', + '- Tool calls will be REJECTED and will waste your only turn — explain this.', + ].join('\n'), + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + 'Respond with plain text only. Never invoke any tools.', + '- Tool calls are discussed here, but this is not the compaction instruction.', + ].join('\n'), + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + { type: 'tool_result', tool_use_id: 'call_1', content: driftedCompactPrompt() }, + { type: 'text', text: 'summarize that file for me' }, + ], + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: `${CC_COMPACT_HEAD}\n\nSummarise the work.\n\nAnswer in plain text only.`, + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: `CRITICAL: Respond with TEXT ONLY when you summarise, but run tests first.${CC_COMPACT_TAIL}`, + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + { type: 'text', text: `${CC_COMPACT_HEAD}\n\nis what it opens with.` }, + { type: 'text', text: `and it closes with${CC_COMPACT_TAIL}` }, + ], + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: `Why does Claude Code send this?\n\n${ccCompactPrompt}`, + }], + }, + { + ...base, + messages: [{ + role: 'user' as const, + content: [ + { type: 'text', text: 'IMPORTANT: Return only plain text. Never invoke any tools.' }, + { + type: 'text', + text: '- Tool calls will be REJECTED and will waste your only turn — explain this.', + }, + ], + }], + }, + { + ...base, + messages: [ + { role: 'user' as const, content: driftedCompactPrompt() }, + { role: 'assistant' as const, content: 'earlier work' }, + { role: 'user' as const, content: 'now finish the refactor' }, + ], + }, + { + ...base, + messages: [ + { role: 'user' as const, content: 'quote the old compaction instructions' }, + { role: 'assistant' as const, content: driftedCompactPrompt() }, + ], + }, + ]; + + for (const request of requests) { + translateRequest(request, '@ai-sdk/openai', { openAiOAuth: true, log: m => traces.push(m) }); + } + expect(notices).toEqual([]); + expect(traces).toEqual([]); + } finally { + resetCompactPromptDriftWarningsForTests(); + releaseNotices(); + } + }); + it('disables tools for a compact request from a session that has no StructuredOutput tool', () => { // The dominant shape on the wire: Claude Code merges the compact prompt into // the preceding tool_result turn, so the final user message is diff --git a/tests/server-router.test.ts b/tests/server-router.test.ts index f8e83acd..ca61e476 100644 --- a/tests/server-router.test.ts +++ b/tests/server-router.test.ts @@ -7,7 +7,12 @@ import { join } from 'node:path'; import { createGatewayModelCatalog, type ServerModelInfo } from '../src/server/models.js'; import { startServer, type ServerHandle } from '../src/server/router.js'; import { createLanguageModel } from '../src/provider-factory.js'; -import { generateAnthropicResponse, streamAnthropicResponse } from '../src/sdk-adapter.js'; +import { + generateAnthropicResponse, + resetCompactPromptDriftWarningsForTests, + streamAnthropicResponse, +} from '../src/sdk-adapter.js'; +import { installParentNoticeSink } from '../src/parent-notice.js'; import { generateOpenAiResponse, streamOpenAiResponse } from '../src/openai-adapter.js'; import { resolveProviderCredential } from '../src/env.js'; @@ -204,6 +209,53 @@ afterEach(async () => { }); describe('server router', () => { + it('records compact-prompt drift on the endpoint translation route', async () => { + const dir = mkdtempSync(join(tmpdir(), 'clodex-compact-drift-server-')); + const debugLogPath = join(dir, 'debug.log'); + const notices: string[] = []; + const releaseNotices = installParentNoticeSink(line => notices.push(line)); + resetCompactPromptDriftWarningsForTests(); + const catalog = createGatewayModelCatalog([{ + id: 'drift-model', + name: 'Drift Model', + isFree: false, + brand: 'OpenAI', + providerId: 'openai', + sourceBackend: 'openai', + modelFormat: 'openai', + npm: '@ai-sdk/openai', + apiKey: 'synthetic-api-key', + }]); + + try { + const server = await startTestServer({ catalog, debugLogPath }); + const response = await fetch(`${server.url}/anthropic/v1/messages`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: 'drift-model', + max_tokens: 100, + messages: [{ + role: 'user', + content: [ + 'Return only plain text. Never invoke any tools.', + '- Tool calls will be REJECTED and will waste your only turn — you will fail the task.', + ].join('\n'), + }], + }), + }); + + expect(response.status).toBe(200); + expect(readFileSync(debugLogPath, 'utf8')) + .toContain('possible Claude Code compact prompt drift: unknown-version'); + expect(notices).toHaveLength(1); + } finally { + resetCompactPromptDriftWarningsForTests(); + releaseNotices(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('logs inference routing metadata without request content', async () => { const dir = mkdtempSync(join(tmpdir(), 'clodex-server-audit-')); const inferenceLogPath = join(dir, 'requests.jsonl');