Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions SHA256-MANIFEST.json
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"apps/server/src/loaf.ts": "e2c85948ffd62a06b70d1af3131972c1d391eb4526d2a4f8ce291d001bece995",
"apps/server/src/mirror-controls.ts": "fd49534b398b2cd79e82c500fc6c3c1a7a27299f7e06c696c6aae118d840f9d7",
"apps/server/src/native-credentials.ts": "42571cefa80fd1132aae6c073904b83d9eb85c0c1e346e89863b926819ab02f7",
"apps/server/src/openai.ts": "2e8a33ac7aa21e7fc850b74b0425e2d08f51bfe10a5509c529b9d56a01d09857",
"apps/server/src/openai.ts": "3526d5762688ae1de98873da20605b19292b920b8ce5fa02cd97f0c1bac2669a",
"apps/server/src/openapi-document.ts": "dfeffe737ec95029835cc1a10f78b3763de94db64eb53256fef90d460eaa999b",
"apps/server/src/pngspeak.ts": "8bf9317f0d804a865d1341753650d4c1433d1c1751f0fb0a216ea7137519f338",
"apps/server/src/preflight.ts": "127a62a56231dcfd2c7bf5a3331f09cb1ffc7e0b07a2663245acb81086ce8fbb",
Expand Down Expand Up @@ -92,7 +92,7 @@
"apps/server/tests/local-route-auth.test.mjs": "9b1aba28b17ac7a856eaac88b6bc8288fdf8a8b4ea74e8f77098b50db31eacb6",
"apps/server/tests/native-credentials.test.mjs": "ab370b33461c7bcdad88dc0e32d8dffe7bd823f91eabc768ee1609352f00fe11",
"apps/server/tests/openai-edit-history.test.mjs": "2daadede6a7d6dd13ebe2d1eadd6487746eb1521a33d6b51979eb116f1da48c6",
"apps/server/tests/openai-routes.test.mjs": "8d1b946454d8b2f49f1b76dd4c8a5564a368d9e897d220269935262e937a63cf",
"apps/server/tests/openai-routes.test.mjs": "119491bf867d840755ecaca926348a5719ff0c6aacbcbf293fffb05dcfade19a",
"apps/server/tests/openai-stream-lifecycle.test.mjs": "997eaf33d452f68b8a11c86424379bb3b37cd2ccb64d550549fd3149763ae02d",
"apps/server/tests/openai-tool-calls.test.mjs": "1a9d518d4f98d2770004d004aacb75d50cd480ecb64329d54d0cc90b9093da9f",
"apps/server/tests/openapi-contract.test.mjs": "4c1e3427a34026aca4c794a7f598eef3e4d91657a115027bf7be9d324e67a857",
Expand Down
33 changes: 27 additions & 6 deletions apps/server/src/openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -484,6 +484,27 @@ export function remainingStreamText(responseText: string, emittedText: string):
return responseText.slice(emittedText.length);
}

/** Return the longest prefix that cannot become part of an unresolved file reference. */
export function safeStreamingTextLength(text: string): number {
let boundary = text.length;
const markers = ["sandbox:", "file-service:", "sediment:"];
for (const marker of markers) {
const index = text.indexOf(marker);
if (index >= 0) boundary = Math.min(boundary, index);
// Keep a partial marker at the end of a snapshot until the next update,
// so a marker split across upstream SSE events is never sent as plain text.
for (let length = Math.min(marker.length - 1, text.length); length > 0; length--) {
if (text.endsWith(marker.slice(0, length))) {
boundary = Math.min(boundary, text.length - length);
break;
}
}
}
const privateMarker = text.search(/[\uE000-\uF8FF]/);
if (privateMarker >= 0) boundary = Math.min(boundary, privateMarker);
return boundary;
Comment on lines +503 to +505

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Hold unfinished Markdown links before emitting their labels

When an upstream snapshot ends with Here is [report]( and a later snapshot appends sandbox:/mnt/data/report.csv), this helper emits the entire first snapshot because it contains no recognized marker. Once the destination arrives, renderRichOutput replaces the whole Markdown link with Download file: [report.csv](...), including the already-emitted label. Consequently, remainingStreamText throws Upstream changed text already delivered to the client, terminating both Chat Completions and Responses streams instead of delivering the download. The previous bracket guard prevented this case; retain unfinished link syntax until its destination is known rather than buffering only the pointer prefix.

Useful? React with 👍 / 👎.

}

export { CompletionBody, OpenAiMessage, ContentPart };
export type { TextPart };

Expand Down Expand Up @@ -1054,13 +1075,13 @@ export async function registerOpenAiRoutes(
? event.text.slice(previous.length)
: (streamedText ? "\n\n" : "") + event.text;
streamedText += append;
// Keep the unfinished line and reference-bearing suffix until
// late file metadata has arrived. SSE heartbeats remain active.
// Plain answer text can be sent as soon as it arrives. Hold
// only unresolved file-reference markers until their metadata
// is available; waiting for a newline made ordinary one-line
// answers look like non-streaming responses.
if (!rich) {
const boundary = streamedText.lastIndexOf("\n") + 1;
const reference = streamedText.search(/\[|[\uE000-\uF8FF]|sandbox:|file-service:|sediment:/);
const safe = streamedText.slice(0, reference < 0 ? boundary : Math.min(boundary, reference));
const delta = safe.slice(emittedText.length);
const safeLength = safeStreamingTextLength(streamedText);
const delta = streamedText.slice(emittedText.length, safeLength);
if (delta) {
if (responses) responses.delta(delta);
else sse(reply, { id: completionId, object: "chat.completion.chunk", created, model: body.model,
Expand Down
96 changes: 96 additions & 0 deletions apps/server/tests/openai-routes.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -1942,6 +1942,14 @@ test("sse() destroys the connection once the backlog stays stalled past the time
]);
});

test("streamable text holds split file references but does not wait for a newline", () => {
assert.equal(openai.safeStreamingTextLength("A short answer without a newline"), "A short answer without a newline".length);
assert.equal(openai.safeStreamingTextLength("A [bracketed phrase] is ordinary text"), "A [bracketed phrase] is ordinary text".length);
assert.equal(openai.safeStreamingTextLength("Download: file-ser"), "Download: ".length);
assert.equal(openai.safeStreamingTextLength("Download: file-service://file-1"), "Download: ".length);
assert.equal(openai.safeStreamingTextLength("Answer \uE200citation"), "Answer ".length);
});

// ---------------------------------------------------------------------------
// Real-socket-only branches: light-my-request's mock socket (used by every
// app.inject() call above) has no setNoDelay method, so the guard added
Expand Down Expand Up @@ -1988,6 +1996,94 @@ test("a streaming completion over a real socket still calls setNoDelay and compl
);
});

test("Chat Completions and Responses emit ordinary answer text before the upstream stream ends", async () => {
const realFetch = globalThis.fetch;
for (const endpoint of ["chat/completions", "responses"]) {
const account = `account-early-stream-${endpoint}`;
useSession(account);
let upstreamController;
let upstreamClosed = false;
const upstreamBody = new ReadableStream({
start(controller) { upstreamController = controller; },
});
const backend = stubBackend(account);
const handler = async (url, init) => {
if (new URL(String(url)).pathname.endsWith("/f/conversation"))
return new Response(upstreamBody, { headers: { "content-type": "text/event-stream" } });
return backend(url, init);
};
await withFetch(handler, async () => {
const realApp = Fastify({ bodyLimit: 30 * 1024 * 1024 });
await openai.registerOpenAiRoutes(realApp);
await realApp.listen({ port: 0, host: "127.0.0.1" });
try {
const address = realApp.server.address();
const port = typeof address === "object" && address ? address.port : address;
const requestBody = endpoint === "responses"
? { model: "auto", stream: true, input: "Say hello" }
: { model: "auto", stream: true, messages: [{ role: "user", content: "Say hello" }] };
const response = await realFetch(`http://127.0.0.1:${port}/v1/${endpoint}`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(requestBody),
});
assert.equal(response.status, 200);
const reader = response.body.getReader();
const firstFrame = `data: ${JSON.stringify(assistantAddFrame("upstream-early", "assistant-early", "First words arrive"))}\n\n`;
upstreamController.enqueue(new TextEncoder().encode(firstFrame));

let pending = "";
const readUntilText = async () => {
const decoder = new TextDecoder();
while (true) {
const { done, value } = await reader.read();
if (done) return false;
pending += decoder.decode(value, { stream: true });
const frames = pending.split(/\r?\n\r?\n/);
pending = frames.pop() ?? "";
for (const frame of frames) {
const dataLine = frame.split(/\r?\n/).find(line => line.startsWith("data: "));
if (!dataLine) continue;
const event = JSON.parse(dataLine.slice(6));
const text = endpoint === "responses"
? event.type === "response.output_text.delta" ? event.delta : ""
: event.choices?.[0]?.delta?.content ?? "";
if (text.includes("First words arrive")) return true;
}
}
};
const early = await Promise.race([
readUntilText().then(value => ({ value })),
new Promise(resolve => setTimeout(() => resolve({ timeout: true }), 1000)),
]);
assert.deepEqual(early, { value: true }, `${endpoint} did not emit text before upstream EOF`);
assert.equal(upstreamClosed, false);

upstreamController.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ p: "/message/content/parts/0", o: "append", v: " and the rest." })}\n\ndata: [DONE]\n\n`));
upstreamClosed = true;
upstreamController.close();
const remainder = await reader.read();
// The consumer may still have buffered the terminal events, so drain
// the response before asserting that completion was delivered.
let tail = remainder.done ? "" : new TextDecoder().decode(remainder.value);
while (true) {
const next = await reader.read();
if (next.done) break;
tail += new TextDecoder().decode(next.value);
}
assert.match(tail, endpoint === "responses" ? /response\.completed/ : /\[DONE\]/);
await reader.cancel().catch(() => undefined);
} finally {
if (!upstreamClosed) {
upstreamClosed = true;
try { upstreamController?.close(); } catch { /* already closed */ }
}
await realApp.close();
}
});
}
});

test("a client that disconnects mid-stream aborts the in-flight upstream turn", async () => {
useSession("account-branch-disconnect");
const realFetch = globalThis.fetch;
Expand Down
Loading