From b145fb8bd202ecf69e7eeaef08bcb6e862de433f Mon Sep 17 00:00:00 2001 From: mahdiwafy Date: Wed, 22 Jul 2026 14:43:30 +0700 Subject: [PATCH 1/2] fix(usage): parse Antigravity/Gemini SSE format (response.candidates + usageMetadata) in parseSSEToOpenAIResponse The parseSSEToOpenAIResponse function only understood OpenAI SSE format (choices[0].delta.content, root-level usage). Antigravity/Gemini SSE chunks use a different schema: data: {"response":{"candidates":[{...}],"usageMetadata":{...}}} - Text content from response.candidates[0].content.parts[].text - Reasoning from .parts[].thought - Finish reason from candidate.finishReason - Usage from response.usageMetadata (promptTokenCount, candidatesTokenCount, thoughtsTokenCount, totalTokenCount) Also moves the OpenAI usage extraction before the AG block so the existing path is preserved for OpenAI-formatted chunks. --- .../handlers/chatCore/sseToJsonHandler.js | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/open-sse/handlers/chatCore/sseToJsonHandler.js b/open-sse/handlers/chatCore/sseToJsonHandler.js index 09383971be..ab4ace087c 100644 --- a/open-sse/handlers/chatCore/sseToJsonHandler.js +++ b/open-sse/handlers/chatCore/sseToJsonHandler.js @@ -76,11 +76,43 @@ export function parseSSEToOpenAIResponse(rawSSE, fallbackModel, validToolNames = for (const chunk of chunks) { const choice = chunk?.choices?.[0]; const delta = choice?.delta || {}; + + // OpenAI format: content in delta + usage at chunk root if (typeof delta.content === "string" && delta.content.length > 0) contentParts.push(delta.content); if (typeof delta.reasoning_content === "string" && delta.reasoning_content.length > 0) reasoningParts.push(delta.reasoning_content); if (choice?.finish_reason) finishReason = choice.finish_reason; if (chunk?.usage && typeof chunk.usage === "object") usage = chunk.usage; + // Gemini / Antigravity format: content in response.candidates[*].content.parts + // and usageMetadata inside response envelope + // data: {"response":{"candidates":[{...}],"usageMetadata":{"promptTokenCount":...,...}}} + if (!choice && chunk.response) { + const resp = chunk.response; + const candidate = resp.candidates?.[0]; + if (candidate) { + const parts = candidate.content?.parts || []; + for (const part of parts) { + if (part.text !== undefined && part.text) contentParts.push(part.text); + if (part.thought && part.text) reasoningParts.push(part.text); + } + if (candidate.finishReason) finishReason = candidate.finishReason.toLowerCase(); + } + // Extract usage from Gemini AG envelope + const usageMeta = resp.usageMetadata || chunk.usageMetadata; + if (usageMeta && !usage) { + const prompt = usageMeta.promptTokenCount || 0; + const completion = usageMeta.candidatesTokenCount || 0; + usage = { + prompt_tokens: prompt, + completion_tokens: completion, + total_tokens: usageMeta.totalTokenCount || (prompt + completion), + completion_tokens_details: { + reasoning_tokens: usageMeta.thoughtsTokenCount || 0 + } + }; + } + } + // Accumulate tool_calls from streaming deltas if (Array.isArray(delta.tool_calls)) { for (const tc of delta.tool_calls) { From 0e88dbe0a3eb3e52470284f34708688b0c4b35af Mon Sep 17 00:00:00 2001 From: mahdiwafy Date: Fri, 24 Jul 2026 13:27:57 +0700 Subject: [PATCH 2/2] fix(usage): count tool_calls toward completion tokens in streaming usage tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tool-call-only turns (agentic coding clients like Claude Code/Kiro sending 100+ tools) stream their real output entirely through delta.tool_calls (and delta.partial_json for Claude tool_use), with delta.content and delta.reasoning_content empty. totalContentLength in createSSEStream() only summed content/reasoning length, so these turns left it stuck at 0. That starved both usage fallbacks: - estimateUsage() (provider sent no/invalid usage at all) - the new hasZeroCompletionWithContent()/fixZeroCompletionUsage() path (provider sent completion_tokens: 0 despite real output) Net effect: usage logs showed "out=0" for most tool-heavy turns even though the client received a large tool_calls payload — matching the live claude-sonnet-5 report on VansRouter (a5a5b570 account/Shiteru). Changes: - stream.js: sum delta.tool_calls[].function.name/arguments length (both PASSTHROUGH and TRANSLATE modes) and delta.partial_json (Claude tool_use) into totalContentLength. - usageTracking.js: add hasZeroCompletionWithContent() and fixZeroCompletionUsage() to patch a provider-reported zero completion count without discarding its (usually accurate) prompt_tokens. - Wire the new fallback into all 4 usage-decision sites in stream.js. Verified: reproduced the exact bug shape in a new integration test (in=166912, completion_tokens: 0 finish chunk, pure tool_calls output) — now estimates a non-zero completion count instead of logging 0. Deployed the fix live to the running container and confirmed in production logs: claude-sonnet-5 tool-heavy turns now log out=44/56/322/etc instead of a long run of out=0. Separately observed (NOT fixed by this change, needs its own investigation): a distinct pattern of genuinely empty completions (response.content === "[Empty streaming response]", no tool_calls either) from the Shiteru-backed claude-sonnet-5 route once prompt size exceeds ~130k tokens — confirmed via requestDetails table, not a usage- counting artifact. --- open-sse/utils/stream.js | 52 ++++++- open-sse/utils/usageTracking.js | 53 ++++++++ .../unit/shiteru-zero-completion-fix.test.js | 64 +++++++++ .../unit/stream-tool-call-only-usage.test.js | 128 ++++++++++++++++++ 4 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 tests/unit/shiteru-zero-completion-fix.test.js create mode 100644 tests/unit/stream-tool-call-only-usage.test.js diff --git a/open-sse/utils/stream.js b/open-sse/utils/stream.js index 94225107e7..6119c0ede1 100644 --- a/open-sse/utils/stream.js +++ b/open-sse/utils/stream.js @@ -1,7 +1,7 @@ import { translateResponse, initState } from "../translator/index.js"; import { FORMATS } from "../translator/formats.js"; import { trackPendingRequest, appendRequestLog } from "@/lib/usageDb.js"; -import { extractUsage, mergeUsage, hasValidUsage, estimateUsage, logUsage, addBufferToUsage, filterUsageForFormat, COLORS } from "./usageTracking.js"; +import { extractUsage, mergeUsage, hasValidUsage, estimateUsage, logUsage, addBufferToUsage, filterUsageForFormat, hasZeroCompletionWithContent, fixZeroCompletionUsage, COLORS } from "./usageTracking.js"; import { parseSSELine, hasValuableContent, fixInvalidId, formatSSE } from "./streamHelpers.js"; import { getOpenAIResponsesEventName, isOpenAIResponsesTerminalEvent, formatIncompleteOpenAIResponsesStreamFailure } from "./responsesStreamHelpers.js"; import { dbg, isDebugEnabled } from "./debugLog.js"; @@ -230,6 +230,17 @@ export function createSSEStream(options = {}) { totalContentLength += reasoning.length; accumulatedThinking += reasoning; } + // Tool-call-only responses (e.g. agentic coding tools with large + // tool arrays) produce real output entirely via delta.tool_calls, + // with content/reasoning empty. Count function name + streamed + // argument chunk length so totalContentLength isn't stuck at 0 + // and the zero-completion / estimation fallbacks below can fire. + if (Array.isArray(delta?.tool_calls)) { + for (const tc of delta.tool_calls) { + if (typeof tc?.function?.name === "string") totalContentLength += tc.function.name.length; + if (typeof tc?.function?.arguments === "string") totalContentLength += tc.function.arguments.length; + } + } const extracted = extractUsage(parsed); if (extracted) { @@ -243,6 +254,15 @@ export function createSSEStream(options = {}) { output = `data: ${JSON.stringify(parsed)}\n`; usage = estimated; injectedUsage = true; + } else if (isFinishChunk && hasZeroCompletionWithContent(parsed.usage, totalContentLength)) { + // Provider reported completion_tokens: 0 (e.g. Shiteru "estimated" + // finish chunk on a large prompt) despite real streamed content. + // Keep the provider's prompt_tokens, patch only the output side. + const fixed = fixZeroCompletionUsage(parsed.usage, totalContentLength); + parsed.usage = filterUsageForFormat(fixed, FORMATS.OPENAI); + output = `data: ${JSON.stringify(parsed)}\n`; + usage = fixed; + injectedUsage = true; } else if (isFinishChunk && usage) { const buffered = addBufferToUsage(usage); parsed.usage = filterUsageForFormat(buffered, FORMATS.OPENAI); @@ -322,12 +342,26 @@ export function createSSEStream(options = {}) { totalContentLength += parsed.delta.thinking.length; accumulatedThinking += parsed.delta.thinking; } + // Claude format - tool_use input streamed as partial_json deltas. + // Tool-call-only turns would otherwise leave totalContentLength at 0. + if (typeof parsed.delta?.partial_json === "string") { + totalContentLength += parsed.delta.partial_json.length; + } // OpenAI format - content if (parsed.choices?.[0]?.delta?.content) { totalContentLength += parsed.choices[0].delta.content.length; accumulatedContent += parsed.choices[0].delta.content; } + // OpenAI format - tool calls (name + streamed argument chunks). Without + // this, tool-call-only turns leave totalContentLength at 0 and the + // zero-completion/estimation usage fallbacks never fire. + if (Array.isArray(parsed.choices?.[0]?.delta?.tool_calls)) { + for (const tc of parsed.choices[0].delta.tool_calls) { + if (typeof tc?.function?.name === "string") totalContentLength += tc.function.name.length; + if (typeof tc?.function?.arguments === "string") totalContentLength += tc.function.arguments.length; + } + } // Detect and correct native Kimi tool-call markup that leaks into the // content stream instead of being emitted as structured tool_calls. @@ -429,6 +463,13 @@ export function createSSEStream(options = {}) { const estimated = estimateUsage(body, totalContentLength, sourceFormat); item.usage = filterUsageForFormat(estimated, sourceFormat); // Filter + already has buffer state.usage = estimated; + } else if (state.finishReason && isFinishChunk && hasZeroCompletionWithContent(item.usage, totalContentLength)) { + // Provider reported zero completion tokens despite real streamed + // content (e.g. Shiteru-style "estimated" usage on large prompts). + // Patch just the output side, keep the provider's prompt_tokens. + const fixed = fixZeroCompletionUsage(item.usage, totalContentLength); + item.usage = filterUsageForFormat(fixed, sourceFormat); + state.usage = fixed; } else if (state.finishReason && isFinishChunk && state.usage) { // Add buffer and filter usage for client (but keep original in state.usage for logging) const buffered = addBufferToUsage(state.usage); @@ -464,6 +505,11 @@ export function createSSEStream(options = {}) { if (!hasValidUsage(usage) && totalContentLength > 0) { usage = estimateUsage(body, totalContentLength, FORMATS.OPENAI); + } else if (hasZeroCompletionWithContent(usage, totalContentLength)) { + // Provider (e.g. Shiteru on large prompts) reported completion_tokens: 0 + // despite real streamed content. hasValidUsage() above passes because + // prompt_tokens is non-zero, so patch just the output side here. + usage = fixZeroCompletionUsage(usage, totalContentLength); } if (hasValidUsage(usage)) { @@ -570,6 +616,10 @@ export function createSSEStream(options = {}) { if (!hasValidUsage(state?.usage) && totalContentLength > 0) { state.usage = estimateUsage(body, totalContentLength, sourceFormat); + } else if (hasZeroCompletionWithContent(state?.usage, totalContentLength)) { + // Provider reported completion_tokens: 0 despite real streamed content. + // hasValidUsage() above passes because prompt_tokens is non-zero. + state.usage = fixZeroCompletionUsage(state.usage, totalContentLength); } if (hasValidUsage(state?.usage)) { diff --git a/open-sse/utils/usageTracking.js b/open-sse/utils/usageTracking.js index 098447e7fa..3d270944af 100644 --- a/open-sse/utils/usageTracking.js +++ b/open-sse/utils/usageTracking.js @@ -227,6 +227,59 @@ export function hasValidUsage(usage) { return false; } +/** + * Detect a usage object with a provider-reported zero completion count + * despite real streamed content having been produced. + * + * Some providers (e.g. Shiteru / OpenAI-compatible passthrough on large prompts) + * return a finish chunk like: + * {"usage": {"prompt_tokens": 182812, "completion_tokens": 0, "estimated": true}} + * + * hasValidUsage() returns true because prompt_tokens > 0, so the caller's + * "estimate the whole thing from scratch" fallback is skipped — yet + * completion_tokens is 0 even though the client received real content. + * Distinct from hasValidUsage(): this only flags the zero-completion-with- + * content case, so callers can patch just the output side and keep the + * provider's (usually more accurate) input token count intact. + * + * @param {object|null} usage - Usage object (any format) + * @param {number} totalContentLength - Total length of streamed response content + * @returns {boolean} true if completion_tokens/output_tokens is a zero that contradicts real content + */ +export function hasZeroCompletionWithContent(usage, totalContentLength = 0) { + if (!usage || typeof usage !== "object") return false; + if (!(totalContentLength > 0)) return false; + + const outTokens = usage.completion_tokens ?? usage.output_tokens; + return typeof outTokens === "number" && outTokens === 0; +} + +/** + * Patch a usage object whose completion/output tokens are zero despite real + * streamed content, replacing only the output side with an estimate derived + * from content length. Keeps the provider's prompt/input token count intact + * (it's usually accurate — only the provider's output estimate failed). + * + * @param {object} usage - Usage object with completion_tokens/output_tokens === 0 + * @param {number} totalContentLength - Total length of streamed response content + * @returns {object} New usage object with corrected completion/total tokens + */ +export function fixZeroCompletionUsage(usage, totalContentLength) { + const estimatedOut = estimateOutputTokens(totalContentLength); + const fixed = { ...usage, estimated: true }; + + if (fixed.completion_tokens !== undefined) { + fixed.completion_tokens = estimatedOut; + const prompt = fixed.prompt_tokens || 0; + fixed.total_tokens = prompt + estimatedOut; + } + if (fixed.output_tokens !== undefined) { + fixed.output_tokens = estimatedOut; + } + + return fixed; +} + /** * Extract usage from any format (Claude, OpenAI, Gemini, Responses API) */ diff --git a/tests/unit/shiteru-zero-completion-fix.test.js b/tests/unit/shiteru-zero-completion-fix.test.js new file mode 100644 index 0000000000..047cb06547 --- /dev/null +++ b/tests/unit/shiteru-zero-completion-fix.test.js @@ -0,0 +1,64 @@ +// Regression test: Shiteru / OpenAI-compatible passthrough on large prompts +// returns a finish chunk with completion_tokens: 0 despite real streamed +// content ("estimated": true, output estimation failed upstream). +// hasValidUsage() alone treats this as valid because prompt_tokens > 0, so +// the router must detect and patch the zero-completion case separately. +import { describe, it, expect } from "vitest"; +import { + hasValidUsage, + hasZeroCompletionWithContent, + fixZeroCompletionUsage, +} from "../../open-sse/utils/usageTracking.js"; + +describe("hasZeroCompletionWithContent", () => { + it("flags OpenAI-shape usage with completion_tokens: 0 when content was streamed", () => { + const usage = { prompt_tokens: 182812, completion_tokens: 0, total_tokens: 182812, estimated: true }; + expect(hasValidUsage(usage)).toBe(true); // this is exactly the trap: passes hasValidUsage + expect(hasZeroCompletionWithContent(usage, 1200)).toBe(true); + }); + + it("flags Claude-shape usage with output_tokens: 0 when content was streamed", () => { + const usage = { input_tokens: 50000, output_tokens: 0 }; + expect(hasZeroCompletionWithContent(usage, 500)).toBe(true); + }); + + it("does not flag when completion_tokens is legitimately non-zero", () => { + const usage = { prompt_tokens: 1000, completion_tokens: 42, total_tokens: 1042 }; + expect(hasZeroCompletionWithContent(usage, 200)).toBe(false); + }); + + it("does not flag zero completion_tokens when no content was actually streamed", () => { + const usage = { prompt_tokens: 1000, completion_tokens: 0 }; + expect(hasZeroCompletionWithContent(usage, 0)).toBe(false); + }); + + it("returns false for null/undefined usage", () => { + expect(hasZeroCompletionWithContent(null, 500)).toBe(false); + expect(hasZeroCompletionWithContent(undefined, 500)).toBe(false); + }); +}); + +describe("fixZeroCompletionUsage", () => { + it("patches OpenAI-shape usage: keeps prompt_tokens, estimates completion_tokens, recomputes total", () => { + const usage = { prompt_tokens: 182812, completion_tokens: 0, total_tokens: 182812, estimated: true }; + const fixed = fixZeroCompletionUsage(usage, 1200); // ~300 estimated tokens (1200/4) + expect(fixed.prompt_tokens).toBe(182812); // provider's real input count preserved + expect(fixed.completion_tokens).toBe(300); + expect(fixed.total_tokens).toBe(183112); + expect(fixed.estimated).toBe(true); + expect(hasValidUsage(fixed)).toBe(true); + }); + + it("patches Claude-shape usage: keeps input_tokens, estimates output_tokens", () => { + const usage = { input_tokens: 50000, output_tokens: 0 }; + const fixed = fixZeroCompletionUsage(usage, 400); // 100 estimated tokens + expect(fixed.input_tokens).toBe(50000); + expect(fixed.output_tokens).toBe(100); + }); + + it("minimum 1 estimated output token when content is non-empty", () => { + const usage = { prompt_tokens: 100, completion_tokens: 0 }; + const fixed = fixZeroCompletionUsage(usage, 1); // 1 char -> floor(1/4)=0, but min 1 + expect(fixed.completion_tokens).toBeGreaterThanOrEqual(1); + }); +}); diff --git a/tests/unit/stream-tool-call-only-usage.test.js b/tests/unit/stream-tool-call-only-usage.test.js new file mode 100644 index 0000000000..1152891af7 --- /dev/null +++ b/tests/unit/stream-tool-call-only-usage.test.js @@ -0,0 +1,128 @@ +// Regression test for the "claude-sonnet-5 ... 0↓" bug reported live: +// agentic tool-call-only turns (e.g. Claude Code / builder combo with 200+ +// tools) stream their real output entirely through delta.tool_calls, with +// delta.content and delta.reasoning_content empty. totalContentLength in +// createSSEStream() only summed content/reasoning length, so tool-call-only +// turns left it at 0 — which starves both the "estimate from scratch" +// fallback and the zero-completion-with-content fallback, and the provider's +// own completion_tokens: 0 (or absent) usage was logged as-is: out=0, despite +// the client receiving a large tool_calls payload. +import { describe, it, expect, vi } from "vitest"; +import { createSSEStream } from "../../open-sse/utils/stream.js"; +import { FORMATS } from "../../open-sse/translator/formats.js"; + +function sseChunk(obj) { + return new TextEncoder().encode(`data: ${JSON.stringify(obj)}\n\n`); +} + +async function runPassthroughStream(chunks, { body } = {}) { + let finalUsage = null; + let finalContent = null; + + const stream = createSSEStream({ + mode: "passthrough", + provider: "openai-compatible-chat-test", + model: "claude-sonnet-5", + connectionId: "conn-test", + body: body || { model: "claude-sonnet-5", messages: [{ role: "user", content: "x" }], tools: [] }, + onStreamComplete: (content, usage) => { + finalContent = content; + finalUsage = usage; + }, + }); + + const readable = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(c); + controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n")); + controller.close(); + }, + }); + + const transformed = readable.pipeThrough(stream); + const reader = transformed.getReader(); + // Drain the output stream so the flush() path (which calls onStreamComplete) runs. + // eslint-disable-next-line no-constant-condition + while (true) { + const { done } = await reader.read(); + if (done) break; + } + + return { finalUsage, finalContent }; +} + +describe("createSSEStream passthrough — tool-call-only usage tracking", () => { + it("counts tool_calls argument length toward totalContentLength (no valid usage from provider)", async () => { + const toolCallChunk = sseChunk({ + id: "chatcmpl-1", + choices: [{ + index: 0, + delta: { + tool_calls: [{ + index: 0, + id: "call_1", + type: "function", + function: { name: "read_file", arguments: JSON.stringify({ path: "/a/b.txt" }) }, + }], + }, + }], + }); + const finishChunk = sseChunk({ + id: "chatcmpl-1", + choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], + // No usage field at all — provider omitted it entirely (worse than 0). + }); + + const { finalUsage } = await runPassthroughStream([toolCallChunk, finishChunk]); + + expect(finalUsage).not.toBeNull(); + expect(finalUsage.completion_tokens).toBeGreaterThan(0); + expect(finalUsage.estimated).toBe(true); + }); + + it("patches provider-reported completion_tokens: 0 when output was pure tool_calls (the exact live bug)", async () => { + const toolCallChunk = sseChunk({ + id: "chatcmpl-2", + choices: [{ + index: 0, + delta: { + tool_calls: [{ + index: 0, + id: "call_2", + type: "function", + function: { name: "edit_file", arguments: JSON.stringify({ path: "/x.js", content: "a".repeat(2000) }) }, + }], + }, + }], + }); + const finishChunk = sseChunk({ + id: "chatcmpl-2", + choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], + // Provider-shape from the bug report: real prompt_tokens, completion_tokens: 0. + usage: { prompt_tokens: 166912, completion_tokens: 0, total_tokens: 166912 }, + }); + + const { finalUsage } = await runPassthroughStream([toolCallChunk, finishChunk]); + + expect(finalUsage).not.toBeNull(); + expect(finalUsage.prompt_tokens).toBe(166912); // provider's real input count preserved + expect(finalUsage.completion_tokens).toBeGreaterThan(0); // no longer stuck at 0 + }); + + it("plain text-only responses still work (no regression for the common case)", async () => { + const textChunk = sseChunk({ + id: "chatcmpl-3", + choices: [{ index: 0, delta: { content: "Hello world, this is a normal reply." } }], + }); + const finishChunk = sseChunk({ + id: "chatcmpl-3", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 500, completion_tokens: 10, total_tokens: 510 }, + }); + + const { finalUsage } = await runPassthroughStream([textChunk, finishChunk]); + + expect(finalUsage.prompt_tokens).toBe(500); + expect(finalUsage.completion_tokens).toBe(10); // untouched, was already valid+non-zero + }); +});