diff --git a/src/adapters/devin.ts b/src/adapters/devin.ts index 9260976058..ab7abfa0a9 100644 --- a/src/adapters/devin.ts +++ b/src/adapters/devin.ts @@ -160,15 +160,44 @@ function assistantToolCalls(message: OcxAssistantMessage): Array<{ id: string; n function assistantText(message: OcxAssistantMessage): string { return message.content - // Thinking stays out of the replayed content. Cognition has no reasoning - // replay field, and folding chain-of-thought into assistant text sends it - // back as visible prior output - which the model then treats as something - // it said to the user. + // Thinking stays out of the replayed TEXT: folding chain-of-thought into + // assistant text sends it back as visible prior output, which the model + // then treats as something it said to the user. It is replayed in its own + // field instead — see assistantThinking below. .map((part) => (part.type === "text" ? part.text : "")) .filter(Boolean) .join("\n"); } +/** + * The assistant turn's own reasoning, for replay in ChatMessagePrompt #11. + * + * This adapter previously asserted that Cognition has no reasoning-replay + * field and dropped the thinking outright, so a reasoning model restarted its + * chain on every turn of a tool loop. The field exists: two independent + * clients of the same service write #11 thinking with #12 signature and #18 + * signature_type on the assistant prompt. + * + * The signature attests the thinking it was produced with, so a block without + * one contributes its text and nothing else rather than borrowing a neighbour's. + */ +function assistantThinking( + message: OcxAssistantMessage, +): { thinking?: string; signature?: string } { + const blocks = message.content.filter( + (part): part is Extract => part.type === "thinking", + ); + if (blocks.length === 0) return {}; + const thinking = blocks.map(b => b.thinking).filter(Boolean).join("\n"); + // Only one signature can ride the prompt, so take the last block that has + // one: that is the block the turn actually ended on. + const signature = blocks.filter(b => b.signature).at(-1)?.signature; + return { + ...(thinking ? { thinking } : {}), + ...(signature ? { signature } : {}), + }; +} + export function mapOcxMessagesToDevin(parsed: OcxParsedRequest): ChatHistoryItem[] { const items: ChatHistoryItem[] = []; // Cognition is not an OpenAI host, and this adapter does advertise a real @@ -203,11 +232,15 @@ function mapOneMessage(message: OcxMessage): ChatHistoryItem | undefined { if (message.role === "assistant") { const toolCalls = assistantToolCalls(message); const text = assistantText(message); - if (!text && toolCalls.length === 0) return undefined; + const reasoning = assistantThinking(message); + // A turn that produced only reasoning is still worth replaying: dropping it + // is what makes the next turn re-derive the same chain. + if (!text && toolCalls.length === 0 && !reasoning.thinking) return undefined; return { role: "assistant", content: text || "", ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}), + ...reasoning, }; } if (message.role === "toolResult") { @@ -335,6 +368,12 @@ export function createDevinAdapter( if (event.text) emit({ type: "thinking_delta", thinking: event.text }); continue; } + if (event.kind === "reasoning_signature") { + // Carried back out so the next turn can replay it in the prompt's + // signature field; an unsigned replay is what the service ignores. + emit({ type: "thinking_signature", signature: event.signature }); + continue; + } if (event.kind === "tool_call_start") { closeOpenTool(); openToolId = event.id; diff --git a/src/adapters/devin/cloud-direct/chat.ts b/src/adapters/devin/cloud-direct/chat.ts index d67a7696e1..3e31e97c5c 100644 --- a/src/adapters/devin/cloud-direct/chat.ts +++ b/src/adapters/devin/cloud-direct/chat.ts @@ -120,6 +120,9 @@ export function allocateCascadeId(): string { * #4 num_tokens: int (rough estimate) * #5 safe_for_code_telemetry: bool (1 = ok to log) * #10 images: repeated ImageData (multimodal) + * #11 thinking: string (assistant reasoning, replayed) + * #12 signature: string (opaque attestation for #11) + * #18 signature_type: string * } * * ImageData (exa.codeium_common_pb.ImageData) { @@ -153,7 +156,13 @@ function encodeChatToolCall(tc: { id: string; name: string; arguments: string }) function encodeChatMessagePrompt( content: ContentPart[], source: number, - opts?: { toolCallId?: string; toolCalls?: Array<{ id: string; name: string; arguments: string }> }, + opts?: { + toolCallId?: string; + toolCalls?: Array<{ id: string; name: string; arguments: string }>; + thinking?: string; + signature?: string; + signatureType?: string; + }, ): Buffer { const textParts = content.filter((p): p is { type: 'text'; text: string } => p.type === 'text'); const imageParts = content.filter((p): p is { type: 'image'; mimeType: string; base64Data: string; caption?: string } => p.type === 'image'); @@ -178,6 +187,14 @@ function encodeChatMessagePrompt( for (const img of imageParts) { parts.push(encodeMessage(10, encodeImageData(img))); } + // Reasoning replay. This adapter used to assert that Cognition has no + // reasoning-replay field and drop the assistant's own thinking, so a + // reasoning model restarted its chain on every turn of a tool loop. Two + // independent clients of the same service write it here: #11 thinking, + // #12 signature, #18 signature_type on the assistant prompt. + if (opts?.thinking) parts.push(encodeString(11, opts.thinking)); + if (opts?.signature) parts.push(encodeString(12, opts.signature)); + if (opts?.signatureType) parts.push(encodeString(18, opts.signatureType)); return Buffer.concat(parts); } @@ -342,6 +359,15 @@ export interface ChatHistoryItem { * each ChatToolCall has #1 id, #2 name, #3 arguments_json). */ tool_calls?: Array<{ id: string; name: string; arguments: string }>; + /** + * For `role: 'assistant'` only — the model's own reasoning from that turn, + * replayed so a reasoning model does not restart its chain on the next one. + * Encoded as ChatMessagePrompt #11 with its #12 signature and #18 + * signature_type. + */ + thinking?: string; + signature?: string; + signature_type?: string; } /** @@ -399,6 +425,12 @@ export interface ToolDef { export type CloudChatEvent = | { kind: 'text'; text: string } | { kind: 'reasoning'; text: string } + /** + * `delta_signature` (#10) — the opaque attestation for the reasoning this + * turn produced. Without decoding it there is nothing to put in the prompt's + * #12 on the next turn, so the replay would always be unsigned. + */ + | { kind: 'reasoning_signature'; signature: string } | { kind: 'tool_call_start'; id: string; name: string } | { kind: 'tool_call_args'; @@ -592,6 +624,9 @@ function buildGetChatMessageRequest(args: BuildArgs): Buffer { { toolCallId: m.role === 'tool' ? m.tool_call_id : undefined, toolCalls: m.role === 'assistant' ? m.tool_calls : undefined, + thinking: m.role === 'assistant' ? m.thinking : undefined, + signature: m.role === 'assistant' ? m.signature : undefined, + signatureType: m.role === 'assistant' ? m.signature_type : undefined, }, ), ), @@ -716,6 +751,9 @@ export function* decodeChatFrame(proto: Buffer): Generator { // block instead of inline with the answer. const s = (f.value as Buffer).toString('utf8'); if (s) yield { kind: 'reasoning', text: s }; + } else if (f.num === 10 && f.wire === 2 && Buffer.isBuffer(f.value)) { + const s = (f.value as Buffer).toString('utf8'); + if (s) yield { kind: 'reasoning_signature', signature: s }; } else if (f.num === 6 && f.wire === 2 && Buffer.isBuffer(f.value)) { let id: string | undefined; let name: string | undefined; diff --git a/tests/providers/devin-hardening.test.ts b/tests/providers/devin-hardening.test.ts index 6d85a61007..319b133061 100644 --- a/tests/providers/devin-hardening.test.ts +++ b/tests/providers/devin-hardening.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { normalizeDevinModelId } from "../../src/adapters/devin"; +import { mapOcxMessagesToDevin } from "../../src/adapters/devin"; import { parseDevinAuthPaste, refreshDevinToken } from "../../src/oauth/devin"; import { DEVIN_DEFAULT_API_SERVER, resolveDevinApiBaseUrl, validateDevinApiBaseUrl } from "../../src/oauth/devin/api-base"; import { registerUser } from "../../src/oauth/devin/register-user"; @@ -444,3 +445,76 @@ describe("devin status classification across the newly reachable trailer codes", expect(cls(504)).toEqual({ status: 504, retryable: true }); }); }); + +describe("devin reasoning replay", () => { + const parsedWith = (messages: unknown[]) => ({ + context: { messages, tools: undefined, systemPrompt: undefined }, + options: { toolChoice: undefined }, + }) as never; + function uvarint(value: number): number[] { + const out: number[] = []; + let v = value; + do { const b = v & 0x7f; v = Math.floor(v / 128); out.push(v > 0 ? b | 0x80 : b); } while (v > 0); + return out; + } + function lenDelim(num: number, payload: Buffer): Buffer { + return Buffer.concat([Buffer.from([...uvarint((num << 3) | 2), ...uvarint(payload.length)]), payload]); + } + function fieldsOf(buf: Buffer): Record { + const out: Record = {}; + for (const f of iterFields(buf)) { + if (Buffer.isBuffer(f.value)) (out[f.num] ??= []).push(f.value); + } + return out; + } + + test("an assistant turn's thinking and signature ride the prompt instead of being dropped", () => { + // The adapter used to assert this field did not exist and drop the chain, + // so a reasoning model re-derived it on every turn of a tool loop. + const history = mapOcxMessagesToDevin(parsedWith([ + { role: "user", content: [{ type: "text", text: "hi" }] }, + { + role: "assistant", + content: [ + { type: "thinking", thinking: "step one", signature: "sig-abc" }, + { type: "text", text: "answer" }, + ], + }, + ])); + const assistant = history.find(m => m.role === "assistant"); + expect(assistant?.thinking).toBe("step one"); + expect(assistant?.signature).toBe("sig-abc"); + // Reasoning must not leak into the visible text. + expect(assistant?.content).toBe("answer"); + }); + + test("a turn that produced only reasoning is still replayed", () => { + const history = mapOcxMessagesToDevin(parsedWith([ + { role: "user", content: [{ type: "text", text: "hi" }] }, + { role: "assistant", content: [{ type: "thinking", thinking: "only thought" }] }, + ])); + expect(history.find(m => m.role === "assistant")?.thinking).toBe("only thought"); + }); + + test("the encoded prompt carries thinking at #11 and its signature at #12", () => { + const req = buildGetChatMessageRequestForTests({ + apiKey: "devin-session-token$x", + modelUid: "swe-2", + messages: [ + { role: "user", content: "hi" }, + { role: "assistant", content: "answer", thinking: "step one", signature: "sig-abc" }, + ], + cascadeId: "c", + } as never); + const prompts = fieldsOf(req)[3] ?? []; + const assistantPrompt = prompts.map(fieldsOf).find(p => p[11]); + expect(assistantPrompt?.[11]?.[0]?.toString("utf8")).toBe("step one"); + expect(assistantPrompt?.[12]?.[0]?.toString("utf8")).toBe("sig-abc"); + }); + + test("the response signature is decoded so there is something to replay", () => { + const frame = lenDelim(10, Buffer.from("sig-from-cloud", "utf8")); + const events = [...decodeChatFrame(frame)]; + expect(events).toEqual([{ kind: "reasoning_signature", signature: "sig-from-cloud" }]); + }); +});