From 7a0b5f97806b5306453cc43db7d0abc030305aad Mon Sep 17 00:00:00 2001 From: Jeremy Carter Date: Mon, 16 Nov 2026 21:30:32 -0500 Subject: [PATCH] Stream plain answer text as it arrives --- SHA256-MANIFEST.json | 4 +- apps/server/src/openai.ts | 33 ++++++-- apps/server/tests/openai-routes.test.mjs | 96 ++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 8 deletions(-) diff --git a/SHA256-MANIFEST.json b/SHA256-MANIFEST.json index f4a154e..7d27eba 100644 --- a/SHA256-MANIFEST.json +++ b/SHA256-MANIFEST.json @@ -50,7 +50,7 @@ "apps/server/src/loaf.ts": "e2c85948ffd62a06b70d1af3131972c1d391eb4526d2a4f8ce291d001bece995", "apps/server/src/mirror-controls.ts": "fd49534b398b2cd79e82c500fc6c3c1a7a27299f7e06c696c6aae118d840f9d7", "apps/server/src/native-credentials.ts": "42571cefa80fd1132aae6c073904b83d9eb85c0c1e346e89863b926819ab02f7", - "apps/server/src/openai.ts": "2e8a33ac7aa21e7fc850b74b0425e2d08f51bfe10a5509c529b9d56a01d09857", + "apps/server/src/openai.ts": "3526d5762688ae1de98873da20605b19292b920b8ce5fa02cd97f0c1bac2669a", "apps/server/src/openapi-document.ts": "dfeffe737ec95029835cc1a10f78b3763de94db64eb53256fef90d460eaa999b", "apps/server/src/pngspeak.ts": "8bf9317f0d804a865d1341753650d4c1433d1c1751f0fb0a216ea7137519f338", "apps/server/src/preflight.ts": "127a62a56231dcfd2c7bf5a3331f09cb1ffc7e0b07a2663245acb81086ce8fbb", @@ -92,7 +92,7 @@ "apps/server/tests/local-route-auth.test.mjs": "9b1aba28b17ac7a856eaac88b6bc8288fdf8a8b4ea74e8f77098b50db31eacb6", "apps/server/tests/native-credentials.test.mjs": "ab370b33461c7bcdad88dc0e32d8dffe7bd823f91eabc768ee1609352f00fe11", "apps/server/tests/openai-edit-history.test.mjs": "2daadede6a7d6dd13ebe2d1eadd6487746eb1521a33d6b51979eb116f1da48c6", - "apps/server/tests/openai-routes.test.mjs": "8d1b946454d8b2f49f1b76dd4c8a5564a368d9e897d220269935262e937a63cf", + "apps/server/tests/openai-routes.test.mjs": "119491bf867d840755ecaca926348a5719ff0c6aacbcbf293fffb05dcfade19a", "apps/server/tests/openai-stream-lifecycle.test.mjs": "997eaf33d452f68b8a11c86424379bb3b37cd2ccb64d550549fd3149763ae02d", "apps/server/tests/openai-tool-calls.test.mjs": "1a9d518d4f98d2770004d004aacb75d50cd480ecb64329d54d0cc90b9093da9f", "apps/server/tests/openapi-contract.test.mjs": "4c1e3427a34026aca4c794a7f598eef3e4d91657a115027bf7be9d324e67a857", diff --git a/apps/server/src/openai.ts b/apps/server/src/openai.ts index c7d226f..1f196a9 100644 --- a/apps/server/src/openai.ts +++ b/apps/server/src/openai.ts @@ -484,6 +484,27 @@ export function remainingStreamText(responseText: string, emittedText: string): return responseText.slice(emittedText.length); } +/** Return the longest prefix that cannot become part of an unresolved file reference. */ +export function safeStreamingTextLength(text: string): number { + let boundary = text.length; + const markers = ["sandbox:", "file-service:", "sediment:"]; + for (const marker of markers) { + const index = text.indexOf(marker); + if (index >= 0) boundary = Math.min(boundary, index); + // Keep a partial marker at the end of a snapshot until the next update, + // so a marker split across upstream SSE events is never sent as plain text. + for (let length = Math.min(marker.length - 1, text.length); length > 0; length--) { + if (text.endsWith(marker.slice(0, length))) { + boundary = Math.min(boundary, text.length - length); + break; + } + } + } + const privateMarker = text.search(/[\uE000-\uF8FF]/); + if (privateMarker >= 0) boundary = Math.min(boundary, privateMarker); + return boundary; +} + export { CompletionBody, OpenAiMessage, ContentPart }; export type { TextPart }; @@ -1054,13 +1075,13 @@ export async function registerOpenAiRoutes( ? event.text.slice(previous.length) : (streamedText ? "\n\n" : "") + event.text; streamedText += append; - // Keep the unfinished line and reference-bearing suffix until - // late file metadata has arrived. SSE heartbeats remain active. + // Plain answer text can be sent as soon as it arrives. Hold + // only unresolved file-reference markers until their metadata + // is available; waiting for a newline made ordinary one-line + // answers look like non-streaming responses. if (!rich) { - const boundary = streamedText.lastIndexOf("\n") + 1; - const reference = streamedText.search(/\[|[\uE000-\uF8FF]|sandbox:|file-service:|sediment:/); - const safe = streamedText.slice(0, reference < 0 ? boundary : Math.min(boundary, reference)); - const delta = safe.slice(emittedText.length); + const safeLength = safeStreamingTextLength(streamedText); + const delta = streamedText.slice(emittedText.length, safeLength); if (delta) { if (responses) responses.delta(delta); else sse(reply, { id: completionId, object: "chat.completion.chunk", created, model: body.model, diff --git a/apps/server/tests/openai-routes.test.mjs b/apps/server/tests/openai-routes.test.mjs index 3400eee..3827636 100644 --- a/apps/server/tests/openai-routes.test.mjs +++ b/apps/server/tests/openai-routes.test.mjs @@ -1942,6 +1942,14 @@ test("sse() destroys the connection once the backlog stays stalled past the time ]); }); +test("streamable text holds split file references but does not wait for a newline", () => { + assert.equal(openai.safeStreamingTextLength("A short answer without a newline"), "A short answer without a newline".length); + assert.equal(openai.safeStreamingTextLength("A [bracketed phrase] is ordinary text"), "A [bracketed phrase] is ordinary text".length); + assert.equal(openai.safeStreamingTextLength("Download: file-ser"), "Download: ".length); + assert.equal(openai.safeStreamingTextLength("Download: file-service://file-1"), "Download: ".length); + assert.equal(openai.safeStreamingTextLength("Answer \uE200citation"), "Answer ".length); +}); + // --------------------------------------------------------------------------- // Real-socket-only branches: light-my-request's mock socket (used by every // app.inject() call above) has no setNoDelay method, so the guard added @@ -1988,6 +1996,94 @@ test("a streaming completion over a real socket still calls setNoDelay and compl ); }); +test("Chat Completions and Responses emit ordinary answer text before the upstream stream ends", async () => { + const realFetch = globalThis.fetch; + for (const endpoint of ["chat/completions", "responses"]) { + const account = `account-early-stream-${endpoint}`; + useSession(account); + let upstreamController; + let upstreamClosed = false; + const upstreamBody = new ReadableStream({ + start(controller) { upstreamController = controller; }, + }); + const backend = stubBackend(account); + const handler = async (url, init) => { + if (new URL(String(url)).pathname.endsWith("/f/conversation")) + return new Response(upstreamBody, { headers: { "content-type": "text/event-stream" } }); + return backend(url, init); + }; + await withFetch(handler, async () => { + const realApp = Fastify({ bodyLimit: 30 * 1024 * 1024 }); + await openai.registerOpenAiRoutes(realApp); + await realApp.listen({ port: 0, host: "127.0.0.1" }); + try { + const address = realApp.server.address(); + const port = typeof address === "object" && address ? address.port : address; + const requestBody = endpoint === "responses" + ? { model: "auto", stream: true, input: "Say hello" } + : { model: "auto", stream: true, messages: [{ role: "user", content: "Say hello" }] }; + const response = await realFetch(`http://127.0.0.1:${port}/v1/${endpoint}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(requestBody), + }); + assert.equal(response.status, 200); + const reader = response.body.getReader(); + const firstFrame = `data: ${JSON.stringify(assistantAddFrame("upstream-early", "assistant-early", "First words arrive"))}\n\n`; + upstreamController.enqueue(new TextEncoder().encode(firstFrame)); + + let pending = ""; + const readUntilText = async () => { + const decoder = new TextDecoder(); + while (true) { + const { done, value } = await reader.read(); + if (done) return false; + pending += decoder.decode(value, { stream: true }); + const frames = pending.split(/\r?\n\r?\n/); + pending = frames.pop() ?? ""; + for (const frame of frames) { + const dataLine = frame.split(/\r?\n/).find(line => line.startsWith("data: ")); + if (!dataLine) continue; + const event = JSON.parse(dataLine.slice(6)); + const text = endpoint === "responses" + ? event.type === "response.output_text.delta" ? event.delta : "" + : event.choices?.[0]?.delta?.content ?? ""; + if (text.includes("First words arrive")) return true; + } + } + }; + const early = await Promise.race([ + readUntilText().then(value => ({ value })), + new Promise(resolve => setTimeout(() => resolve({ timeout: true }), 1000)), + ]); + assert.deepEqual(early, { value: true }, `${endpoint} did not emit text before upstream EOF`); + assert.equal(upstreamClosed, false); + + upstreamController.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ p: "/message/content/parts/0", o: "append", v: " and the rest." })}\n\ndata: [DONE]\n\n`)); + upstreamClosed = true; + upstreamController.close(); + const remainder = await reader.read(); + // The consumer may still have buffered the terminal events, so drain + // the response before asserting that completion was delivered. + let tail = remainder.done ? "" : new TextDecoder().decode(remainder.value); + while (true) { + const next = await reader.read(); + if (next.done) break; + tail += new TextDecoder().decode(next.value); + } + assert.match(tail, endpoint === "responses" ? /response\.completed/ : /\[DONE\]/); + await reader.cancel().catch(() => undefined); + } finally { + if (!upstreamClosed) { + upstreamClosed = true; + try { upstreamController?.close(); } catch { /* already closed */ } + } + await realApp.close(); + } + }); + } +}); + test("a client that disconnects mid-stream aborts the in-flight upstream turn", async () => { useSession("account-branch-disconnect"); const realFetch = globalThis.fetch;