diff --git a/src/translator/chat-to-responses.test.ts b/src/translator/chat-to-responses.test.ts index af58871..e380c26 100644 --- a/src/translator/chat-to-responses.test.ts +++ b/src/translator/chat-to-responses.test.ts @@ -27,6 +27,24 @@ describe("chatCompletionsToResponses (batch)", () => { expect((r.output[0] as { content: { type: string; text: string }[] }).content[0]).toEqual({ type: "output_text", text: "hello" }); }); + it("carries cache-inclusive Chat usage into the Responses usage details", () => { + const r = chatCompletionsToResponses(chatResp({ + usage: { + prompt_tokens: 1224, + completion_tokens: 3, + total_tokens: 1227, + prompt_tokens_details: { cached_tokens: 1216 }, + }, + }), "glm-5.2"); + + expect(r.usage).toEqual({ + input_tokens: 1224, + output_tokens: 3, + total_tokens: 1227, + input_tokens_details: { cached_tokens: 1216 }, + }); + }); + it("emits reasoning output item before message", () => { const r = chatCompletionsToResponses(chatResp({ choices: [{ diff --git a/src/translator/openai-to-anthropic.test.ts b/src/translator/openai-to-anthropic.test.ts index af9b81b..248d92e 100644 --- a/src/translator/openai-to-anthropic.test.ts +++ b/src/translator/openai-to-anthropic.test.ts @@ -6,10 +6,12 @@ import { describe, it, expect } from "bun:test"; import { translateRequestOpenAIToAnthropic, translateResponseAnthropicToOpenAI, + anthropicUsageToOpenAI, } from "./openai-to-anthropic.js"; import { translateRequestAnthropicToOpenAI, translateResponseOpenAIToAnthropic, + openaiUsageToAnthropic, } from "./anthropic-to-openai.js"; import type { OpenAIChatRequest, @@ -466,6 +468,76 @@ describe("translateRequestOpenAIToAnthropic", () => { }); }); +describe("anthropicUsageToOpenAI", () => { + it("folds cache reads into prompt_tokens and reports them as cached_tokens", () => { + expect(anthropicUsageToOpenAI({ + input_tokens: 8, + output_tokens: 3, + cache_read_input_tokens: 1216, + cache_creation_input_tokens: 0, + })).toEqual({ + prompt_tokens: 1224, + completion_tokens: 3, + total_tokens: 1227, + prompt_tokens_details: { cached_tokens: 1216 }, + }); + }); + + it("counts both cache buckets but exposes only the read bucket", () => { + const usage = anthropicUsageToOpenAI({ + input_tokens: 100, + output_tokens: 5, + cache_read_input_tokens: 20, + cache_creation_input_tokens: 30, + }); + + expect(usage).toEqual({ + prompt_tokens: 150, + completion_tokens: 5, + total_tokens: 155, + prompt_tokens_details: { cached_tokens: 20 }, + }); + expect(usage).not.toHaveProperty("cache_read_input_tokens"); + expect(usage).not.toHaveProperty("cache_creation_input_tokens"); + }); + + it("does not fabricate prompt_tokens_details when no cache bucket is reported", () => { + expect(anthropicUsageToOpenAI({ input_tokens: 10, output_tokens: 2 })).toEqual({ + prompt_tokens: 10, + completion_tokens: 2, + total_tokens: 12, + }); + }); + + it("preserves an explicitly reported zero cache read", () => { + expect(anthropicUsageToOpenAI({ + input_tokens: 10, + output_tokens: 2, + cache_read_input_tokens: 0, + }).prompt_tokens_details).toEqual({ cached_tokens: 0 }); + }); + + it("round-trips fresh input and cache reads through openaiUsageToAnthropic", () => { + const anthropic = { input_tokens: 8, output_tokens: 3, cache_read_input_tokens: 1216 }; + expect(openaiUsageToAnthropic(anthropicUsageToOpenAI(anthropic))).toEqual(anthropic); + }); + + it("round-trips totals but not the cache-creation bucket, which OpenAI cannot express", () => { + const anthropic = { + input_tokens: 8, + output_tokens: 3, + cache_read_input_tokens: 100, + cache_creation_input_tokens: 20, + }; + const back = openaiUsageToAnthropic(anthropicUsageToOpenAI(anthropic)); + + expect(back.input_tokens + (back.cache_read_input_tokens ?? 0) + (back.cache_creation_input_tokens ?? 0)) + .toBe(128); + expect(back.cache_creation_input_tokens).toBeUndefined(); + expect(back.input_tokens).toBe(28); + }); +}); + describe("translateResponseAnthropicToOpenAI", () => { it("extracts text content from response", () => { const resp: AnthropicMessagesResponse = { @@ -535,6 +607,31 @@ describe("translateResponseAnthropicToOpenAI", () => { expect(result.usage!.total_tokens).toBe(150); }); + it("reports cache-inclusive prompt_tokens for a cached response", () => { + const resp: AnthropicMessagesResponse = { + id: "msg_1", + type: "message", + role: "assistant", + content: [{ type: "text", text: "Hi" }], + model: "glm-4.6", + stop_reason: "end_turn", + stop_sequence: null, + usage: { + input_tokens: 8, + output_tokens: 3, + cache_read_input_tokens: 1216, + cache_creation_input_tokens: 0, + }, + }; + + expect(translateResponseAnthropicToOpenAI(resp, "glm-4.6").usage).toEqual({ + prompt_tokens: 1224, + completion_tokens: 3, + total_tokens: 1227, + prompt_tokens_details: { cached_tokens: 1216 }, + }); + }); + it("preserves thinking blocks as OpenAI reasoning_content", () => { const resp: AnthropicMessagesResponse = { id: "msg_1", diff --git a/src/translator/openai-to-anthropic.ts b/src/translator/openai-to-anthropic.ts index 5f8bb7b..d191f24 100644 --- a/src/translator/openai-to-anthropic.ts +++ b/src/translator/openai-to-anthropic.ts @@ -5,6 +5,7 @@ import type { OpenAIChatRequest, OpenAIChatResponse, + OpenAIUsage, OpenAIMessage, OpenAIToolDefinition, AnthropicMessagesRequest, @@ -13,6 +14,7 @@ import type { AnthropicContentBlock, AnthropicToolDefinition, AnthropicThinkingConfig, + AnthropicUsage, } from "./types.js"; import { MODELS } from "../provider/models.js"; @@ -203,6 +205,40 @@ function imageUrlToAnthropicBlock(url: string): AnthropicContentBlock { return { type: "text", text: url }; } +/** + * Convert an Anthropic usage block into OpenAI's cache-inclusive usage + * semantics — the counterpart of `openaiUsageToAnthropic`. + * + * Anthropic reports the fresh (uncached) input in `input_tokens` and keeps the + * three buckets mutually exclusive; OpenAI's `prompt_tokens` is *inclusive* of + * cache hits. So prompt = input + cache_read + cache_creation, with the cache + * read additionally surfaced through the standard `prompt_tokens_details`. + * The Anthropic-style cache fields are deliberately not mirrored onto the + * OpenAI usage object — strict-schema clients reject unknown members. + * + * Totals round-trip exactly; buckets do not. OpenAI has no cache-creation + * member, so a converted-and-converted-back usage reclassifies those tokens as + * fresh input. `prompt_tokens` and `total_tokens` stay correct either way. + */ +export function anthropicUsageToOpenAI(usage: AnthropicUsage | undefined): OpenAIUsage { + const inputTokens = usage?.input_tokens ?? 0; + const outputTokens = usage?.output_tokens ?? 0; + const cacheRead = usage?.cache_read_input_tokens ?? 0; + const cacheCreation = usage?.cache_creation_input_tokens ?? 0; + const promptTokens = inputTokens + cacheRead + cacheCreation; + + return { + prompt_tokens: promptTokens, + completion_tokens: outputTokens, + total_tokens: promptTokens + outputTokens, + // Presence-preserving: an upstream explicitly reporting 0 cache reads stays + // distinguishable from one reporting no cache breakdown at all. + ...(usage?.cache_read_input_tokens != null + ? { prompt_tokens_details: { cached_tokens: cacheRead } } + : {}), + }; +} + /** Translate an Anthropic messages response into an OpenAI chat completion response. */ export function translateResponseAnthropicToOpenAI( resp: AnthropicMessagesResponse, @@ -242,11 +278,7 @@ export function translateResponseAnthropicToOpenAI( }, finish_reason: finishReason, }], - usage: { - prompt_tokens: resp.usage?.input_tokens ?? 0, - completion_tokens: resp.usage?.output_tokens ?? 0, - total_tokens: (resp.usage?.input_tokens ?? 0) + (resp.usage?.output_tokens ?? 0), - }, + usage: anthropicUsageToOpenAI(resp.usage), }; } diff --git a/src/translator/sse-translator.test.ts b/src/translator/sse-translator.test.ts index 2321df0..eb4e41b 100644 --- a/src/translator/sse-translator.test.ts +++ b/src/translator/sse-translator.test.ts @@ -42,6 +42,12 @@ interface ParsedChunk { }; finish_reason: string | null; }>; + usage?: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + prompt_tokens_details?: { cached_tokens?: number }; + }; } function parseChunks(output: string): ParsedChunk[] { @@ -187,6 +193,187 @@ describe("anthropicSseToOpenaiSse", () => { expect(output).toContain('"total_tokens":15'); }); + it("takes the real usage from message_delta when message_start reports zeros", async () => { + const sse = [ + 'event: message_start', + `data: ${JSON.stringify({ + type: "message_start", + message: { id: "msg_1", model: "glm-4.6", usage: { input_tokens: 0, output_tokens: 0 } }, + })}`, + '', + 'event: message_delta', + `data: ${JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { + input_tokens: 8, + output_tokens: 3, + cache_read_input_tokens: 1216, + server_tool_use: { web_search_requests: 0 }, + service_tier: "standard", + }, + })}`, + '', + 'event: message_stop', + `data: ${JSON.stringify({ type: "message_stop" })}`, + '', + ].join('\n'); + + const output = await collectStream(anthropicSseToOpenaiSse(makeStream(sse), "glm-4.6")); + const usageChunks = parseChunks(output).filter((c) => c.usage); + + expect(usageChunks).toHaveLength(1); + expect(usageChunks[0].choices[0].finish_reason).toBe("stop"); + expect(usageChunks[0].usage).toEqual({ + prompt_tokens: 1224, + completion_tokens: 3, + total_tokens: 1227, + prompt_tokens_details: { cached_tokens: 1216 }, + }); + }); + + it("merges successive usage deltas per field, last write wins", async () => { + const sse = [ + 'event: message_start', + `data: ${JSON.stringify({ + type: "message_start", + message: { id: "msg_1", model: "glm-4.6", usage: { input_tokens: 0, output_tokens: 0 } }, + })}`, + '', + 'event: message_delta', + `data: ${JSON.stringify({ + type: "message_delta", + delta: {}, + usage: { input_tokens: 10, cache_read_input_tokens: 1216, cache_creation_input_tokens: 4 }, + })}`, + '', + 'event: message_delta', + `data: ${JSON.stringify({ type: "message_delta", delta: {}, usage: { input_tokens: 8 } })}`, + '', + 'event: message_delta', + `data: ${JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { output_tokens: 3, cache_read_input_tokens: 0 }, + })}`, + '', + 'event: message_stop', + `data: ${JSON.stringify({ type: "message_stop" })}`, + '', + ].join('\n'); + + const output = await collectStream(anthropicSseToOpenaiSse(makeStream(sse), "glm-4.6")); + const usageChunk = parseChunks(output).find((c) => c.usage); + + // input 8 (overwritten), cache_read 0 (a later zero wins), cache_creation 4 (retained). + expect(usageChunk?.usage).toEqual({ + prompt_tokens: 12, + completion_tokens: 3, + total_tokens: 15, + prompt_tokens_details: { cached_tokens: 0 }, + }); + }); + + it("falls back to message_start usage when no message_delta arrives", async () => { + const sse = [ + 'event: message_start', + `data: ${JSON.stringify({ + type: "message_start", + message: { + id: "msg_1", + model: "glm-4.6", + usage: { + input_tokens: 10, + output_tokens: 5, + cache_read_input_tokens: 20, + cache_creation_input_tokens: 2, + }, + }, + })}`, + '', + 'event: message_stop', + `data: ${JSON.stringify({ type: "message_stop" })}`, + '', + ].join('\n'); + + const output = await collectStream(anthropicSseToOpenaiSse(makeStream(sse), "glm-4.6")); + const usageChunk = parseChunks(output).find((c) => c.usage); + + expect(usageChunk?.usage).toEqual({ + prompt_tokens: 32, + completion_tokens: 5, + total_tokens: 37, + prompt_tokens_details: { cached_tokens: 20 }, + }); + }); + + it("accepts a usage-only message_delta that carries no delta member at all", async () => { + const sse = [ + 'event: message_start', + `data: ${JSON.stringify({ type: "message_start", message: { id: "msg_1", model: "glm-4.6" } })}`, + '', + 'event: message_delta', + `data: ${JSON.stringify({ type: "message_delta", usage: { input_tokens: 40, output_tokens: 7 } })}`, + '', + 'event: message_stop', + `data: ${JSON.stringify({ type: "message_stop" })}`, + '', + ].join('\n'); + + const output = await collectStream(anthropicSseToOpenaiSse(makeStream(sse), "glm-4.6")); + const usageChunk = parseChunks(output).find((c) => c.usage); + + expect(usageChunk?.usage).toEqual({ prompt_tokens: 40, completion_tokens: 7, total_tokens: 47 }); + }); + + it("emits a single finish chunk when two message_delta events both carry a stop_reason", async () => { + const sse = [ + 'event: message_start', + `data: ${JSON.stringify({ type: "message_start", message: { id: "msg_1", model: "glm-4.6" } })}`, + '', + 'event: message_delta', + `data: ${JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 40, output_tokens: 7 }, + })}`, + '', + 'event: message_delta', + `data: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "max_tokens" } })}`, + '', + 'event: message_stop', + `data: ${JSON.stringify({ type: "message_stop" })}`, + '', + ].join('\n'); + + const output = await collectStream(anthropicSseToOpenaiSse(makeStream(sse), "glm-4.6")); + const finishChunks = parseChunks(output).filter((c) => c.choices[0]?.finish_reason); + + expect(finishChunks).toHaveLength(1); + expect(finishChunks[0].choices[0].finish_reason).toBe("stop"); + expect(finishChunks[0].usage).toEqual({ prompt_tokens: 40, completion_tokens: 7, total_tokens: 47 }); + }); + + it("still emits a zeroed usage block when the upstream reports none", async () => { + const sse = [ + 'event: message_start', + `data: ${JSON.stringify({ type: "message_start", message: { id: "msg_1", model: "glm-4.6" } })}`, + '', + 'event: message_stop', + `data: ${JSON.stringify({ type: "message_stop" })}`, + '', + ].join('\n'); + + const output = await collectStream(anthropicSseToOpenaiSse(makeStream(sse), "glm-4.6")); + const usageChunk = parseChunks(output).find((c) => c.usage); + + expect(usageChunk?.usage).toEqual({ + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + }); + }); + it("handles max_tokens stop reason", async () => { const sse = [ 'event: message_start', diff --git a/src/translator/sse-translator.ts b/src/translator/sse-translator.ts index b4de412..496dd6e 100644 --- a/src/translator/sse-translator.ts +++ b/src/translator/sse-translator.ts @@ -3,8 +3,9 @@ * @see .omo/plans/zcode-proxy.md Task 12 * @see https://docs.anthropic.com/en/api/messages-streaming */ -import type { AnthropicStreamEvent, OpenAIStreamChunk, OpenAIStreamToolCall, OpenAIUsage } from "./types.js"; +import type { AnthropicStreamEvent, AnthropicUsage, OpenAIStreamChunk, OpenAIStreamToolCall, OpenAIUsage } from "./types.js"; import { openaiUsageToAnthropic } from "./anthropic-to-openai.js"; +import { anthropicUsageToOpenAI } from "./openai-to-anthropic.js"; /** Parse a raw SSE chunk string into event type + JSON data. */ export interface ParsedSSE { @@ -47,8 +48,8 @@ export interface TranslationState { messageId: string; model: string; roleSent: boolean; - inputTokens: number; - outputTokens: number; + /** Running Anthropic usage snapshot; the upstream reports it incrementally. */ + usage: AnthropicUsage; toolCallIndex: number; blockIndexToToolCallIndex: Map; finishReasonSent: boolean; @@ -59,21 +60,34 @@ export function initState(model: string): TranslationState { messageId: "", model, roleSent: false, - inputTokens: 0, - outputTokens: 0, + usage: { input_tokens: 0, output_tokens: 0 }, toolCallIndex: 0, blockIndexToToolCallIndex: new Map(), finishReasonSent: false, }; } +/** + * Fold an incremental usage report into the running snapshot, field by field. + * Overwrite (not accumulate) semantics: the upstream re-reports absolute + * totals, so a later 0 must win over an earlier non-zero, while a field the + * event omits keeps its previous value. + */ +function mergeAnthropicUsage(target: AnthropicUsage, patch: Partial | undefined): void { + if (!patch) return; + if (patch.input_tokens != null) target.input_tokens = patch.input_tokens; + if (patch.output_tokens != null) target.output_tokens = patch.output_tokens; + if (patch.cache_read_input_tokens != null) target.cache_read_input_tokens = patch.cache_read_input_tokens; + if (patch.cache_creation_input_tokens != null) target.cache_creation_input_tokens = patch.cache_creation_input_tokens; +} + function makeChunk( state: TranslationState, delta: Record, finishReason: string | null = null, - usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number }, + usage?: OpenAIUsage, ): string { - const chunk: OpenAIStreamChunk & { usage?: typeof usage } = { + const chunk: OpenAIStreamChunk = { id: state.messageId || "chatcmpl-stream", object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), @@ -160,7 +174,7 @@ export function translateEvent(state: TranslationState, sse: ParsedSSE): string const msg = (data as any).message; state.messageId = msg?.id ?? "msg_stream"; state.model = msg?.model ?? state.model; - state.inputTokens = msg?.usage?.input_tokens ?? 0; + mergeAnthropicUsage(state.usage, msg?.usage); if (!state.roleSent) { state.roleSent = true; return makeChunk(state, { role: "assistant" }); @@ -214,30 +228,25 @@ export function translateEvent(state: TranslationState, sse: ParsedSSE): string } case "message_delta": { - const dataAny = data as any; - const delta = dataAny.delta; - if (dataAny?.usage?.output_tokens !== undefined) { - state.outputTokens = dataAny.usage.output_tokens; - } - if (delta?.stop_reason) { - const finishReason = mapStopReason(delta.stop_reason); + // Fold usage in *before* the stop_reason branch: the upstream carries + // both in the same event, and this is the only place the real input and + // cache counts ever arrive. Usage that lands only *after* the finish + // chunk was emitted updates the snapshot but is not re-announced — the + // alternative is a trailing `choices: []` usage chunk, which some clients + // index into blindly. No observed upstream orders the events that way. + mergeAnthropicUsage(state.usage, data.usage); + if (data.delta?.stop_reason && !state.finishReasonSent) { + const finishReason = mapStopReason(data.delta.stop_reason); state.finishReasonSent = true; - return makeChunk(state, {}, finishReason, { - prompt_tokens: state.inputTokens, - completion_tokens: state.outputTokens, - total_tokens: state.inputTokens + state.outputTokens, - }); + return makeChunk(state, {}, finishReason, anthropicUsageToOpenAI(state.usage)); } return null; } case "message_stop": { if (state.finishReasonSent) return null; - return makeChunk(state, {}, "stop", { - prompt_tokens: state.inputTokens, - completion_tokens: state.outputTokens, - total_tokens: state.inputTokens + state.outputTokens, - }); + state.finishReasonSent = true; + return makeChunk(state, {}, "stop", anthropicUsageToOpenAI(state.usage)); } case "ping": diff --git a/src/translator/types.ts b/src/translator/types.ts index 9b81293..c665658 100644 --- a/src/translator/types.ts +++ b/src/translator/types.ts @@ -255,8 +255,13 @@ export type AnthropicStreamEvent = | { type: "content_block_stop"; index: number } | { type: "message_delta"; - delta: { stop_reason: string; stop_sequence?: string | null }; - usage?: { output_tokens: number }; + delta?: { stop_reason?: string | null; stop_sequence?: string | null }; + /** + * The upstream reports the real token counts here rather than in + * `message_start` — including the input and cache buckets. Extra vendor + * members (`server_tool_use`, `service_tier`) are left unmodelled. + */ + usage?: Partial; } | { type: "message_stop" } | { type: "ping" };