diff --git a/electron/ai-edition/chat-persistence.test.ts b/electron/ai-edition/chat-persistence.test.ts index 782ed579d..5053884ae 100644 --- a/electron/ai-edition/chat-persistence.test.ts +++ b/electron/ai-edition/chat-persistence.test.ts @@ -3,6 +3,8 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createEmptyDocument } from "../../src/lib/ai-edition/schema"; +import { ChatPersistence } from "./chat-persistence"; +import { createTurnUsageAccumulator } from "./deep-agent/usage"; vi.mock("./deep-agent/service", () => ({ invokeOpenScreenAgent: vi.fn() })); vi.mock("./deep-agent/chat-model", () => ({ @@ -58,6 +60,7 @@ beforeEach(() => { text: "reply", document: args.document, mutated: false, + usage: createTurnUsageAccumulator().snapshot(), })); }); @@ -67,6 +70,244 @@ afterEach(async () => { }); describe("local Agent chat history", () => { + const reportedUsage = { + inputTokens: 31, + outputTokens: 7, + totalTokens: 38, + modelCallCount: 2, + countedCallCount: 2, + status: "complete" as const, + cachedInputTokens: 9, + reasoningOutputTokens: 3, + }; + + it.each([true, false])("restores usage on the owning message (success=%s)", async (success) => { + invokeMock.mockImplementation(async (args) => ({ + text: success ? "reply" : "", + reason: success ? undefined : "provider failed", + document: args.document, + mutated: false, + usage: { ...reportedUsage, rawProvider: { apiKey: key } }, + })); + const session = createSession("proj_usage"); + const result = await runChat("proj_usage", session.id, "question", config()); + if (!success) expect(result.userMessage).toMatchObject({ role: "user", usage: reportedUsage }); + await restart(); + const messages = selectSession("proj_usage", session.id)?.messages ?? []; + expect(messages).toHaveLength(success ? 2 : 1); + expect(messages.at(-1)?.usage).toEqual(reportedUsage); + expect(messages.at(-1)?.durationMs).toEqual(expect.any(Number)); + if (success) expect(messages[0]?.usage).toBeUndefined(); + expect(readFileSync(path.join(root, "proj_usage.chat.json"), "utf8")).not.toContain(key); + }); + + it.each([ + true, + false, + ])("binds delayed usage to the original session (success=%s)", async (success) => { + let reply!: (value: Awaited>) => void; + invokeMock.mockImplementation(() => new Promise((resolve) => (reply = resolve))); + const first = createSession("proj_binding", "A"); + const second = createSession("proj_binding", "B"); + const turn = runChat("proj_binding", first.id, "question", config()); + await vi.waitFor(() => expect(invokeMock).toHaveBeenCalled()); + selectSession("proj_binding", second.id); + reply({ + text: success ? "reply" : "", + document: createEmptyDocument({ title: "x", projectId: "proj_binding" }), + mutated: false, + usage: reportedUsage, + }); + await turn; + await restart(); + expect(selectSession("proj_binding", first.id)?.messages.at(-1)?.usage).toEqual(reportedUsage); + expect(selectSession("proj_binding", second.id)?.messages).toEqual([]); + }); + + it.each([ + ["session", true], + ["session", false], + ["project", true], + ["project", false], + ["rewind", true], + ["rewind", false], + ])("discards delayed results after %s removal (success=%s)", async (boundary, success) => { + const projectId = "proj_boundary"; + const session = createSession(projectId); + const document = createEmptyDocument({ title: "x", projectId }); + await runChat(projectId, session.id, "earlier", config(), document); + const earlier = selectSession(projectId, session.id)!.messages[0]; + let reply!: (value: Awaited>) => void; + invokeMock.mockImplementation(() => new Promise((resolve) => (reply = resolve))); + const turn = runChat(projectId, session.id, "removed question", config(), document); + await vi.waitFor(() => expect(invokeMock).toHaveBeenCalledTimes(2)); + if (boundary === "session") deleteSession(projectId, session.id); + if (boundary === "project") { + await deleteProjectChat(projectId); + createSession(projectId, "replacement"); + } + if (boundary === "rewind") + expect(rewindToMessage(projectId, session.id, earlier.id).success).toBe(true); + reply({ text: success ? "late reply" : "", document, mutated: false, usage: reportedUsage }); + expect(await turn).toMatchObject({ success: false }); + await restart(); + const allMessages = listSessions(projectId).flatMap( + (entry) => selectSession(projectId, entry.id)?.messages ?? [], + ); + expect( + allMessages.some( + (entry) => entry.content === "late reply" || entry.content === "removed question", + ), + ).toBe(false); + }); + + it.each([ + true, + false, + ])("discards a pending turn rewound to its own user message (success=%s)", async (success) => { + const projectId = "proj_own_rewind"; + const session = createSession(projectId); + const document = createEmptyDocument({ title: "before", projectId }); + let reply!: (value: Awaited>) => void; + invokeMock.mockImplementation(() => new Promise((resolve) => (reply = resolve))); + const turn = runChat(projectId, session.id, "pending question", config(), document); + await vi.waitFor(() => expect(invokeMock).toHaveBeenCalled()); + const user = selectSession(projectId, session.id)!.messages[0]; + expect(rewindToMessage(projectId, session.id, user.id).success).toBe(true); + reply({ + text: success ? "stale reply" : "", + document: createEmptyDocument({ title: "stale edit", projectId }), + mutated: true, + usage: reportedUsage, + }); + const result = await turn; + expect(result.success).toBe(false); + expect(result.document).toBeUndefined(); + expect(result.assistantMessage).toBeUndefined(); + expect(result.userMessage).toBeUndefined(); + expect(selectSession(projectId, session.id)?.messages).toEqual([]); + await restart(); + const restored = selectSession(projectId, session.id)?.messages; + expect(restored).toEqual([]); + }); + + it("uses the actual session created for a stale id", async () => { + await runChat("proj_stale", "stale-id", "question", config()); + await restart(); + const session = listSessions("proj_stale")[0]; + expect(selectSession("proj_stale", session.id)?.messages[1]?.usage?.status).toBe("unavailable"); + }); + + it("retains legacy messages and drops only invalid usage", async () => { + const session = createSession("proj_corrupt"); + await runChat("proj_corrupt", session.id, "question", config()); + await flushChatPersistence(); + const file = path.join(root, "proj_corrupt.chat.json"); + const saved = JSON.parse(readFileSync(file, "utf8")); + delete saved.sessions[0].messages[0].usage; + saved.sessions[0].messages[1].usage = { ...reportedUsage, countedCallCount: 3 }; + writeFileSync(file, JSON.stringify(saved)); + await restart(); + const messages = selectSession("proj_corrupt", session.id)?.messages; + expect(messages?.map((entry) => entry.content)).toEqual(["question", "reply"]); + expect(messages?.every((entry) => entry.usage === undefined)).toBe(true); + }); + + it.each([ + { inputTokens: -1 }, + { outputTokens: 1.5 }, + { totalTokens: Number.MAX_SAFE_INTEGER + 1 }, + { countedCallCount: 3 }, + { modelCallCount: 0, countedCallCount: 0 }, + { totalTokens: 39 }, + { cachedInputTokens: 32 }, + { cacheCreationInputTokens: 32 }, + { reasoningOutputTokens: 8 }, + { status: "unavailable" as const, countedCallCount: 0 }, + { status: "partial" as const, countedCallCount: 0 }, + ])("drops malformed usage while retaining its message: %j", async (invalid) => { + const store = new ChatPersistence(root); + await store.write("proj_validation", [ + { + id: "session", + projectId: "proj_validation", + title: "Title", + createdAt: "now", + messages: [ + { + id: "message", + role: "assistant", + content: "retained", + createdAt: "now", + usage: { ...reportedUsage, ...invalid }, + }, + ], + }, + ]); + const messages = new ChatPersistence(root).read("proj_validation")[0]?.messages; + expect(messages).toHaveLength(1); + expect(messages?.[0]).toMatchObject({ id: "message", content: "retained" }); + expect(messages?.[0].usage).toBeUndefined(); + }); + + it.each([ + { ...reportedUsage, status: "partial" as const }, + { + ...reportedUsage, + status: "partial" as const, + inputTokens: Number.MAX_SAFE_INTEGER, + totalTokens: 1, + }, + { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + modelCallCount: 1, + countedCallCount: 0, + status: "unavailable" as const, + }, + ])("retains valid partial lower bounds and unavailable usage: %j", async (usage) => { + const store = new ChatPersistence(root); + await store.write("proj_valid", [ + { + id: "session", + projectId: "proj_valid", + title: "Title", + createdAt: "now", + messages: [{ id: "message", role: "assistant", content: "reply", createdAt: "now", usage }], + }, + ]); + expect(new ChatPersistence(root).read("proj_valid")[0]?.messages[0].usage).toEqual(usage); + }); + + it("does not create usage before model validation succeeds", async () => { + const session = createSession("proj_no_config"); + const result = await runChat("proj_no_config", session.id, "question", { + getConfig: () => null, + } as unknown as LlmConfigStore); + expect(result).toMatchObject({ success: false }); + expect(result.userMessage).toBeUndefined(); + expect(invokeMock).not.toHaveBeenCalled(); + expect(selectSession("proj_no_config", session.id)?.messages).toEqual([]); + }); + + it("keeps failure diagnostics and usage out of subsequent model history", async () => { + invokeMock.mockImplementation(async (args) => ({ + text: "", + reason: "diagnostic-secret", + document: args.document, + mutated: false, + usage: reportedUsage, + })); + const session = createSession("proj_failure_history"); + await runChat("proj_failure_history", session.id, "first question", config()); + await runChat("proj_failure_history", session.id, "second question", config()); + expect(invokeMock.mock.lastCall?.[0].history).toEqual([ + { role: "user", content: "first question" }, + { role: "user", content: "second question" }, + ]); + }); + it("restores multiple sessions and persists rename and delete per project", async () => { const first = createSession("proj_a", "First"); const deleted = createSession("proj_a", "Deleted"); @@ -237,7 +478,12 @@ describe("local Agent chat history", () => { await vi.waitFor(() => expect(invokeMock).toHaveBeenCalled()); await deleteProjectChat("proj_mid_turn"); const document = createEmptyDocument({ title: "x", projectId: "proj_mid_turn" }); - reply({ text: "late reply", document, mutated: false }); + reply({ + text: "late reply", + document, + mutated: false, + usage: createTurnUsageAccumulator().snapshot(), + }); expect(await turn).toMatchObject({ success: false }); await restart(); diff --git a/electron/ai-edition/chat-persistence.ts b/electron/ai-edition/chat-persistence.ts index 79d6d9079..1b29f18fd 100644 --- a/electron/ai-edition/chat-persistence.ts +++ b/electron/ai-edition/chat-persistence.ts @@ -10,12 +10,58 @@ import { z } from "zod"; import { renameWithRetry } from "../atomic-rename"; const toolCallSchema = z.object({ name: z.string(), summary: z.string() }); +const tokenCountSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +const usageSchema = z + .object({ + inputTokens: tokenCountSchema, + outputTokens: tokenCountSchema, + totalTokens: tokenCountSchema, + modelCallCount: tokenCountSchema, + countedCallCount: tokenCountSchema, + status: z.enum(["complete", "partial", "unavailable"]), + cachedInputTokens: tokenCountSchema.optional(), + cacheCreationInputTokens: tokenCountSchema.optional(), + reasoningOutputTokens: tokenCountSchema.optional(), + }) + .refine((usage) => { + if (usage.countedCallCount > usage.modelCallCount) return false; + if ((usage.cachedInputTokens ?? 0) > usage.inputTokens) return false; + if ((usage.cacheCreationInputTokens ?? 0) > usage.inputTokens) return false; + if ((usage.reasoningOutputTokens ?? 0) > usage.outputTokens) return false; + if (usage.status === "unavailable") { + return ( + usage.countedCallCount === 0 && + usage.inputTokens === 0 && + usage.outputTokens === 0 && + usage.totalTokens === 0 + ); + } + if (usage.countedCallCount === 0) return false; + if (usage.status === "complete") { + const sum = usage.inputTokens + usage.outputTokens; + return ( + usage.modelCallCount > 0 && + usage.countedCallCount === usage.modelCallCount && + Number.isSafeInteger(sum) && + sum === usage.totalTokens + ); + } + // Partial aggregates retain safe lower bounds independently on overflow. + return true; + }); +// A damaged usage extension must not invalidate an otherwise readable v1 message. +const optionalUsageSchema = z.unknown().transform((value) => { + const parsed = usageSchema.safeParse(value); + return parsed.success ? parsed.data : undefined; +}); const messageSchema = z.object({ id: z.string().min(1), role: z.enum(["user", "assistant"]), content: z.string(), createdAt: z.string().min(1), toolCalls: z.array(toolCallSchema).optional(), + usage: optionalUsageSchema.optional(), + durationMs: tokenCountSchema.optional().catch(undefined), }); const sessionSchema = z.object({ id: z.string().min(1), diff --git a/electron/ai-edition/chat-service.compaction.test.ts b/electron/ai-edition/chat-service.compaction.test.ts index 37f715b35..bee0ff4bc 100644 --- a/electron/ai-edition/chat-service.compaction.test.ts +++ b/electron/ai-edition/chat-service.compaction.test.ts @@ -4,6 +4,7 @@ // behind the summarizer, so no test here needs a provider or a key. import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createTurnUsageAccumulator } from "./deep-agent/usage"; vi.mock("./deep-agent/service", () => ({ invokeOpenScreenAgent: vi.fn(), @@ -60,7 +61,12 @@ beforeEach(() => { chatModelMock.mockReset(); invokeMock.mockImplementation(async (args) => { histories.push([...args.history]); - return { text: "ok", document: args.document, mutated: false }; + return { + text: "ok", + document: args.document, + mutated: false, + usage: createTurnUsageAccumulator().snapshot(), + }; }); }); diff --git a/electron/ai-edition/chat-service.toolloop.test.ts b/electron/ai-edition/chat-service.toolloop.test.ts index 11719179d..0090c56ab 100644 --- a/electron/ai-edition/chat-service.toolloop.test.ts +++ b/electron/ai-edition/chat-service.toolloop.test.ts @@ -10,6 +10,7 @@ import { createEmptyDocument, documentSchema, } from "../../src/lib/ai-edition/schema"; +import { createTurnUsageAccumulator } from "./deep-agent/usage"; vi.mock("./deep-agent/service", () => ({ invokeOpenScreenAgent: vi.fn(), @@ -99,7 +100,12 @@ beforeEach(() => { // ponytail: default mock — tests override per-scenario. invokeMock.mockImplementation(async (args) => { fixture.captured.push({ args }); - return { text: "ok", document: args.document, mutated: false }; + return { + text: "ok", + document: args.document, + mutated: false, + usage: createTurnUsageAccumulator().snapshot(), + }; }); }); @@ -109,7 +115,7 @@ beforeEach(() => { async function streamAgent( args: Parameters[0], events: ReadonlyArray<{ kind: string; payload: unknown }>, -): Promise<{ text: string; document: AxcutDocument; mutated: boolean }> { +): Promise { const sink = args.sink; for (const event of events) { switch (event.kind) { @@ -138,6 +144,7 @@ async function streamAgent( text: "Done.", document: args.document, mutated: false, + usage: createTurnUsageAccumulator().snapshot(), }; } @@ -146,7 +153,12 @@ describe("runChat tool loop", () => { invokeMock.mockImplementationOnce(async (args) => { args.sink.toolStart("addTrim", { startSec: 5, endSec: 8, reason: "silence" }); args.sink.toolEnd("addTrim", true, "added trim 0:05.0 – 0:08.0"); - return { text: "Done.", document: args.document, mutated: true }; + return { + text: "Done.", + document: args.document, + mutated: true, + usage: createTurnUsageAccumulator().snapshot(), + }; }); const session = createSession("proj_loop"); @@ -168,7 +180,12 @@ describe("runChat tool loop", () => { invokeMock.mockImplementationOnce(async (args) => { args.sink.toolStart("addTrim", { startSec: 1, endSec: 2 }); args.sink.toolEnd("addTrim", true, "added trim 0:01.0 – 0:02.0"); - return { text: "Done.", document: args.document, mutated: true }; + return { + text: "Done.", + document: args.document, + mutated: true, + usage: createTurnUsageAccumulator().snapshot(), + }; }); const session = createSession("proj_rewind"); @@ -224,6 +241,7 @@ describe("runChat tool loop", () => { invokeMock.mockImplementationOnce(async (args) => ({ text: "", document: args.document, + usage: createTurnUsageAccumulator().snapshot(), mutated: false, })); const session = createSession("proj_empty_reply"); @@ -273,6 +291,7 @@ describe("runChat tool loop", () => { text: "Hi!", document: args.document, mutated: false, + usage: createTurnUsageAccumulator().snapshot(), })); const s = createSession("proj_text_only"); const result = await runChat("proj_text_only", s.id, "hello", stubConfig()); @@ -291,7 +310,12 @@ describe("runChat tool loop", () => { args.sink.thinking("concluding."); args.sink.toolStart("addTrim", { startSec: 1, endSec: 2 }); args.sink.toolEnd("addTrim", true, "added trim 0:01.0 – 0:02.0"); - return { text: "Done.", document: args.document, mutated: true }; + return { + text: "Done.", + document: args.document, + mutated: true, + usage: createTurnUsageAccumulator().snapshot(), + }; }); const sink = { @@ -335,7 +359,12 @@ describe("runChat tool loop", () => { invokeMock.mockReset(); invokeMock.mockImplementationOnce(async (args) => { args.sink.error("Upstream 404 404 Page not found"); - return { text: "", document: args.document, mutated: false }; + return { + text: "", + document: args.document, + mutated: false, + usage: createTurnUsageAccumulator().snapshot(), + }; }); const sinkErr = { text: (delta: string) => fixture.events.push({ kind: "text", payload: delta }), @@ -402,6 +431,7 @@ describe("allowAgentEdits reaches the agent", () => { text: "I went ahead and cut them.", document: args.document, mutated: true, + usage: createTurnUsageAccumulator().snapshot(), })); const session = createSession("proj_consent_belt"); const result = await runChat( @@ -421,6 +451,7 @@ describe("allowAgentEdits reaches the agent", () => { text: "Cut them.", document: args.document, mutated: true, + usage: createTurnUsageAccumulator().snapshot(), })); const session = createSession("proj_consent_allowed"); const result = await runChat( diff --git a/electron/ai-edition/chat-service.ts b/electron/ai-edition/chat-service.ts index e056f04b1..619e5328c 100644 --- a/electron/ai-edition/chat-service.ts +++ b/electron/ai-edition/chat-service.ts @@ -18,6 +18,7 @@ import { } from "../../src/lib/ai-edition/schema"; import type { ShaderEffectChecker } from "../../src/lib/ai-edition/shaderEffects"; import type { + AgentTurnUsage, AiEditionChatMessage, AiEditionChatResult, AiEditionToolCallSummary, @@ -297,6 +298,7 @@ export function deleteSession(projectId: string, sessionId: string): boolean { } export interface ChatEventSink { + usage?: (usage: AgentTurnUsage) => void; /** Streamed text delta from the model. */ text?: (delta: string) => void; /** Streamed delta from the model's reasoning block (Anthropic/MiniMax @@ -329,6 +331,7 @@ const noop = () => undefined; /** ponytail: zero-config sink that swallows every event. */ const NOOP_SINK: Required = { + usage: noop, text: noop, thinking: noop, toolStart: noop, @@ -434,6 +437,7 @@ export async function runChat( const appliedToolCalls: AiEditionToolCallSummary[] = []; const agentSink = { + usage: (usage: AgentTurnUsage) => emit.usage(usage), text: (delta: string) => emit.text(delta), thinking: (delta: string) => emit.thinking(delta), toolStart: (name: string, args: unknown) => { @@ -450,6 +454,7 @@ export async function runChat( const { invokeOpenScreenAgent } = await import("./deep-agent/service"); + const startedAt = performance.now(); const result = await invokeOpenScreenAgent({ document: workingDocument ?? emptyDocumentForTextOnly(projectId), model: { @@ -469,24 +474,42 @@ export async function runChat( checkShaderEffect: env.checkShaderEffect, }); + const durationMs = Math.max(0, Math.round(performance.now() - startedAt)); + // The project was deleted while the agent ran: its sessions are gone, so a // reply recorded now would be reported as sent and then never be seen again. if (sessionsByProject.get(projectId) !== sessions) { return { success: false, error: "The project was deleted while the agent was replying." }; } + // Bind the turn to the actual session and user message captured above, even + // when the caller supplied a stale id or selected another session meanwhile. + if (sessions.get(session.id) !== session) { + return { success: false, error: "The chat session was deleted while the agent was replying." }; + } + const originatingMessage = session.messages.find((entry) => entry.id === userMessage.id); + // Rewind preserves message ids but replaces survivors with copies. Identity + // also rejects a turn whose own user message was the rewind target. + if (originatingMessage !== userMessage) { + return { success: false, error: "The chat was rewound while the agent was replying." }; + } + if (!result.text) { // ponytail: surface the deep-agent's diagnostic so the user can see // *why* the model produced no text (e.g. MiniMax streaming only // thinking blocks, or a non-text content shape that our extractor // missed). Falls back to the generic message when the agent didn't // provide a reason. + originatingMessage.usage = result.usage; + originatingMessage.durationMs = durationMs; + persistProject(projectId); return { success: false, error: result.reason ?? "Empty response from model.", // The message is stored all the same, and can be rewound to retry it. userMessageId: userMessage.id, userMessageCheckpointId: userMessage.checkpointId ?? undefined, + userMessage: originatingMessage, }; } @@ -494,6 +517,8 @@ export async function runChat( id: randomUUID(), role: "assistant", content: result.text, + usage: result.usage, + durationMs, createdAt: new Date().toISOString(), toolCalls: appliedToolCalls.length ? appliedToolCalls : undefined, }; diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 00fc819ee..1468f83a8 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -47,7 +47,7 @@ import { ZOOM_DEPTH_LEGEND, } from "../../../src/lib/ai-edition/timeline/zoom-scale"; import { ZOOM_TRANSITION_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-transitions"; -import type { SegmentationSupport } from "../../../src/native/contracts"; +import type { AgentTurnUsage, SegmentationSupport } from "../../../src/native/contracts"; import { AGENT_EDITS_SETTING, type AgentToolExecution, @@ -107,8 +107,11 @@ import { messageContentToThinking, type OpenScreenChatModelConfig, } from "./chat-model"; +import { createTurnUsageAccumulator } from "./usage"; export interface OpenScreenAgentSink { + /** Latest cumulative usage from completed model calls; never sum stream chunks. */ + usage?: (usage: AgentTurnUsage) => void; text: (delta: string) => void; /** Streaming delta from the model's reasoning block (Anthropic/MiniMax * thinking). Provider-agnostic — for providers without thinking this is @@ -664,6 +667,7 @@ export async function probeCursorTelemetry( const SILENT_TOOL_EVENTS: ReadonlySet = new Set(["on_tool_start", "on_tool_end"]); export interface InvokeResult { + usage: AgentTurnUsage; text: string; document: AxcutDocument; mutated: boolean; @@ -678,6 +682,9 @@ export async function invokeOpenScreenAgent(args: InvokeArgs): Promise = []; for await (const event of stream) { + usage.observe(event); + if (event.event === "on_chat_model_end" || event.event === "on_chat_model_error") { + sink.usage?.(usage.snapshot()); + } const eventType = typeof event.event === "string" ? event.event : ""; const data = event.data as Record | undefined; const name = typeof event.name === "string" ? (event.name as string) : ""; @@ -784,9 +790,9 @@ export async function invokeOpenScreenAgent(args: InvokeArgs): Promise e.event) .join(",")}). Last chunk: ${sample}`; sink.error(reason); - return { text: "", document: holder.current, mutated, reason }; + return { text: "", document: holder.current, mutated, reason, usage: usage.snapshot() }; } - return { text: finalText.trim(), document: holder.current, mutated }; + return { text: finalText.trim(), document: holder.current, mutated, usage: usage.snapshot() }; } catch (err) { // ponytail: surface the LangChain/HTTP error (with name + truncated // stack) so we can tell whether the stream threw (e.g. MiniMax @@ -810,6 +816,7 @@ export async function invokeOpenScreenAgent(args: InvokeArgs): Promise { + let requests = 0; + const server = createServer((req, res) => { + req.resume(); + req.on("end", () => { + requests++; + if (mode === "failure" && requests === 2) { + res.writeHead(400); + res.end( + JSON.stringify({ error: { message: "fixture failure", type: "invalid_request_error" } }), + ); + return; + } + if (mode === "anthropic") { + sse( + res, + [ + { + type: "message_start", + message: { + id: "msg_1", + type: "message", + role: "assistant", + model: "claude-fixture", + content: [], + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: 10, + output_tokens: 0, + cache_read_input_tokens: 4, + cache_creation_input_tokens: 3, + }, + }, + }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Done" } }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { output_tokens: 5 }, + }, + { type: "message_stop" }, + ], + true, + ); + return; + } + if (mode === "responses") { + const response = { + id: "resp_1", + object: "response", + created_at: 1, + status: "completed", + model: "gpt-fixture", + output: [ + { + id: "m1", + type: "message", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Done", annotations: [] }], + }, + ], + usage: { + input_tokens: 10, + output_tokens: 5, + total_tokens: 15, + input_tokens_details: { cached_tokens: 4 }, + output_tokens_details: { reasoning_tokens: 2 }, + }, + }; + sse( + res, + [ + { + type: "response.created", + response: { ...response, status: "in_progress", output: [] }, + }, + { + type: "response.output_text.delta", + item_id: "m1", + output_index: 0, + content_index: 0, + delta: "Done", + }, + { type: "response.completed", response }, + ], + true, + ); + return; + } + const useTool = (mode === "chat" || mode === "failure") && requests === 1; + const message = useTool + ? { + role: "assistant", + content: "", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "getCurrentDocument", arguments: "{}" }, + }, + ], + } + : { role: "assistant", content: mode === "empty" ? "" : "Done" }; + const usage = { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + prompt_tokens_details: { cached_tokens: 4 }, + completion_tokens_details: { reasoning_tokens: 2 }, + }; + if (mode === "nonstream") { + res.setHeader("content-type", "application/json"); + res.end( + JSON.stringify({ + id: "chat1", + object: "chat.completion", + created: 1, + model: "fixture", + choices: [{ index: 0, message, finish_reason: "stop" }], + usage, + }), + ); + return; + } + sse(res, [ + { + id: "chat1", + object: "chat.completion.chunk", + created: 1, + model: "fixture", + choices: [ + { + index: 0, + delta: useTool + ? { role: "assistant", tool_calls: [{ index: 0, ...message.tool_calls?.[0] }] } + : message, + finish_reason: null, + }, + ], + }, + { + id: "chat1", + object: "chat.completion.chunk", + created: 1, + model: "fixture", + choices: [], + usage, + }, + ]); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("missing fixture address"); + const baseURL = `http://127.0.0.1:${address.port}`; + try { + const model = + mode === "anthropic" + ? new ChatAnthropic({ + apiKey: "fixture", + model: "claude-fixture", + anthropicApiUrl: baseURL, + maxRetries: 0, + }) + : new ChatOpenAI({ + apiKey: "fixture", + model: "fixture", + configuration: { baseURL }, + maxRetries: 0, + useResponsesApi: mode === "responses", + disableStreaming: mode === "nonstream", + }); + const agent = createAgent({ + model, + tools: [ + tool(() => "read", { + name: "getCurrentDocument", + description: "read", + schema: z.object({}), + }), + ], + }); + const accumulator = createTurnUsageAccumulator(); + const ends: Record[] = []; + try { + for await (const event of agent.streamEvents({ + messages: [{ role: "user", content: "Read" }], + })) { + accumulator.observe(event); + if (event.event === "on_chat_model_end") + ends.push(event.data.output as Record); + } + } catch (error) { + if (mode !== "failure") throw error; + } + expect(ends).toHaveLength(mode === "chat" ? 2 : 1); + expect(ends[0].usage_metadata).toMatchObject({ + input_tokens: mode === "anthropic" ? 17 : 10, + output_tokens: 5, + total_tokens: mode === "anthropic" ? 22 : 15, + }); + expect(accumulator.snapshot()).toMatchObject({ + inputTokens: mode === "anthropic" ? 17 : mode === "chat" ? 20 : 10, + outputTokens: mode === "chat" ? 10 : 5, + modelCallCount: mode === "chat" || mode === "failure" ? 2 : 1, + countedCallCount: mode === "chat" ? 2 : 1, + status: mode === "failure" ? "partial" : "complete", + cachedInputTokens: mode === "chat" ? 8 : 4, + }); + if (mode === "anthropic") expect(accumulator.snapshot().cacheCreationInputTokens).toBe(3); + requests = 0; + const usageUpdates = vi.fn(); + const result = await invokeOpenScreenAgent({ + document: createEmptyDocument({ projectId: "usage", title: "usage" }), + model: { + provider: + mode === "anthropic" + ? "anthropic" + : mode === "nonstream" + ? "google" + : "openai-compatible", + model: + mode === "nonstream" + ? "gemini-3-fixture" + : mode === "responses" + ? "gpt-5-fixture" + : "fixture", + apiKey: "fixture", + baseUrl: baseURL, + maxRetries: 0, + }, + history: [], + userMessage: "Read", + sink: { + usage: usageUpdates, + text: vi.fn(), + thinking: vi.fn(), + toolStart: vi.fn(), + toolEnd: vi.fn(), + error: vi.fn(), + }, + }); + expect(usageUpdates).toHaveBeenCalled(); + expect(usageUpdates.mock.lastCall?.[0].totalTokens).toBe(result.usage.totalTokens); + if (mode === "chat") { + expect(usageUpdates.mock.calls.map(([value]) => value.totalTokens)).toEqual([15, 30]); + } + if (mode === "empty") expect(result.reason).toContain("Empty response"); + expect(result.usage).toMatchObject({ + inputTokens: mode === "anthropic" ? 17 : mode === "chat" ? 20 : 10, + outputTokens: mode === "chat" ? 10 : 5, + status: mode === "failure" ? "partial" : "complete", + }); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +}, 15000); + +it("returns unavailable usage for model-construction errors", async () => { + const factory = vi + .spyOn(chatModelFactory, "createOpenScreenChatModel") + .mockRejectedValueOnce(new Error("construction failed")); + const result = await invokeOpenScreenAgent({ + document: createEmptyDocument({ projectId: "usage-error", title: "usage" }), + model: { provider: "openai-compatible", model: "fixture", baseUrl: "not a URL" }, + history: [], + userMessage: "Read", + sink: { + text: vi.fn(), + thinking: vi.fn(), + toolStart: vi.fn(), + toolEnd: vi.fn(), + error: vi.fn(), + }, + }); + expect(result.usage).toMatchObject({ modelCallCount: 0, status: "unavailable" }); + factory.mockRestore(); +}); + +it.each([ + "nonstream-zero", + "stream-zero", + "missing-usage", +])("accounts for %s HTTP metadata", async (mode) => { + const server = createServer((req, res) => { + req.resume(); + req.on("end", () => { + const usage = + mode !== "missing-usage" + ? { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 } + : undefined; + const message = { role: "assistant", content: "Done" }; + if (mode === "nonstream-zero") { + res.setHeader("content-type", "application/json"); + res.end( + JSON.stringify({ + id: "zero", + object: "chat.completion", + created: 1, + model: "fixture", + choices: [{ index: 0, message, finish_reason: "stop" }], + usage, + }), + ); + } else + sse(res, [ + { + id: "missing", + object: "chat.completion.chunk", + created: 1, + model: "fixture", + choices: [{ index: 0, delta: message, finish_reason: "stop" }], + usage, + }, + ]); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("missing address"); + try { + const model = new ChatOpenAI({ + apiKey: "fixture", + model: "fixture", + configuration: { baseURL: `http://127.0.0.1:${address.port}` }, + disableStreaming: mode === "nonstream-zero", + maxRetries: 0, + }); + const agent = createAgent({ model, tools: [] }); + const usage = createTurnUsageAccumulator(); + for await (const event of agent.streamEvents({ + messages: [{ role: "user", content: "Read" }], + })) { + usage.observe(event); + if (event.event === "on_chat_model_end" && mode === "nonstream-zero") { + // Locked ChatOpenAI drops explicit zero completion_tokens from both + // metadata records. Without the raw count, completeness is unknowable. + expect(event.data.output.usage_metadata).toMatchObject({ + input_tokens: 10, + total_tokens: 10, + }); + expect(event.data.output.usage_metadata).not.toHaveProperty("output_tokens"); + expect(event.data.output.response_metadata.tokenUsage).toEqual({ + promptTokens: 10, + totalTokens: 10, + }); + } + } + expect(usage.snapshot()).toMatchObject({ + inputTokens: mode !== "missing-usage" ? 10 : 0, + modelCallCount: 1, + outputTokens: 0, + totalTokens: mode !== "missing-usage" ? 10 : 0, + status: + mode === "nonstream-zero" ? "partial" : mode === "stream-zero" ? "complete" : "unavailable", + }); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +}); diff --git a/electron/ai-edition/deep-agent/usage.test.ts b/electron/ai-edition/deep-agent/usage.test.ts new file mode 100644 index 000000000..a52e59ec1 --- /dev/null +++ b/electron/ai-edition/deep-agent/usage.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, it } from "vitest"; +import { createTurnUsageAccumulator } from "./usage"; + +const start = (run_id = "a") => ({ event: "on_chat_model_start", run_id }); +const end = (usage_metadata?: unknown, run_id = "a") => ({ + event: "on_chat_model_end", + run_id, + data: { output: { usage_metadata } }, +}); +const full = { input_tokens: 10, output_tokens: 5, total_tokens: 15 }; +describe("turn usage", () => { + it.each([ + ["cachedInputTokens", "input", "cache_read"], + ["cacheCreationInputTokens", "input", "cache_creation"], + ["reasoningOutputTokens", "output", "reasoning"], + ] as const)("omits %s when core overflow retains a smaller parent lower bound", (field, direction, detail) => { + const a = createTurnUsageAccumulator(); + const parent = direction === "input" ? "input_tokens" : "output_tokens"; + const details = direction === "input" ? "input_token_details" : "output_token_details"; + a.observe(end({ input_tokens: 0, output_tokens: 0, [parent]: 1, [details]: { [detail]: 1 } })); + a.observe( + end( + { + input_tokens: 0, + output_tokens: 0, + [parent]: Number.MAX_SAFE_INTEGER, + [details]: { [detail]: Number.MAX_SAFE_INTEGER - 1 }, + }, + "b", + ), + ); + const result = a.snapshot(); + expect(result).toMatchObject({ status: "partial", modelCallCount: 2, countedCallCount: 2 }); + expect(direction === "input" ? result.inputTokens : result.outputTokens).toBe(1); + expect(result.totalTokens).toBe(1); + expect(result).not.toHaveProperty(field); + }); + + it("bounds enriched details against retained conflicting core", () => { + const a = createTurnUsageAccumulator(); + a.observe(end(full)); + a.observe( + end({ + input_tokens: 20, + output_tokens: 10, + total_tokens: 30, + input_token_details: { cache_read: 15, cache_creation: 15 }, + output_token_details: { reasoning: 8 }, + }), + ); + expect(a.snapshot()).toMatchObject({ inputTokens: 10, outputTokens: 5, status: "partial" }); + expect(a.snapshot()).not.toHaveProperty("cachedInputTokens"); + expect(a.snapshot()).not.toHaveProperty("cacheCreationInputTokens"); + expect(a.snapshot()).not.toHaveProperty("reasoningOutputTokens"); + }); + + it("enriches missing details on duplicate finals", () => { + const a = createTurnUsageAccumulator(); + a.observe(end(full)); + a.observe(end({ ...full, input_token_details: { cache_read: 4 } })); + expect(a.snapshot()).toMatchObject({ cachedInputTokens: 4, status: "complete" }); + }); + it("preserves first details and marks conflicting details partial", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ ...full, input_token_details: { cache_read: 4 } })); + a.observe(end({ ...full, input_token_details: { cache_read: 8 } })); + expect(a.snapshot()).toMatchObject({ cachedInputTokens: 4, status: "partial" }); + }); + it("merges complementary partial finals", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ input_tokens: 10 })); + a.observe(end({ output_tokens: 5, total_tokens: 15 })); + expect(a.snapshot()).toMatchObject({ + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + status: "complete", + }); + }); + + it("upgrades total-only metadata when later input/output agree", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ total_tokens: 15 })); + a.observe(end({ input_tokens: 10, output_tokens: 5 })); + expect(a.snapshot()).toMatchObject({ totalTokens: 15, status: "complete" }); + }); + + it("sums three completed calls to 660", () => { + const a = createTurnUsageAccumulator(); + for (const [id, input, output] of [ + ["a", 100, 10], + ["b", 200, 20], + ["c", 300, 30], + ] as const) + a.observe(end({ input_tokens: input, output_tokens: output }, id)); + expect(a.snapshot()).toMatchObject({ + inputTokens: 600, + outputTokens: 60, + totalTokens: 660, + modelCallCount: 3, + countedCallCount: 3, + status: "complete", + }); + }); + it("retains optional details when a duplicate end omits them", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ ...full, input_token_details: { cache_read: 4 } })); + a.observe(end(full)); + expect(a.snapshot().cachedInputTokens).toBe(4); + }); + + it("is unavailable without observations or credible usage", () => { + const a = createTurnUsageAccumulator(); + expect(a.snapshot().status).toBe("unavailable"); + a.observe(start()); + a.observe(end()); + expect(a.snapshot()).toMatchObject({ + modelCallCount: 1, + countedCallCount: 0, + status: "unavailable", + }); + }); + it("counts final messages once, ignores chunks and upgrades missing usage", () => { + const a = createTurnUsageAccumulator(); + a.observe(start()); + a.observe({ + event: "on_chat_model_stream", + run_id: "a", + data: { chunk: { usage_metadata: full } }, + }); + a.observe(end()); + a.observe(end(full)); + a.observe(end(full)); + expect(a.snapshot()).toMatchObject({ + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + modelCallCount: 1, + countedCallCount: 1, + status: "complete", + }); + }); + it("retains the first credible record on conflict", () => { + const a = createTurnUsageAccumulator(); + a.observe(end(full)); + a.observe(end({ ...full, input_tokens: 11, total_tokens: 16 })); + expect(a.snapshot()).toMatchObject({ inputTokens: 10, totalTokens: 15, status: "partial" }); + }); + it("upgrades a partial final record when later metadata supplies missing core fields", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ input_tokens: 10 })); + a.observe(end(full)); + expect(a.snapshot()).toMatchObject({ inputTokens: 10, totalTokens: 15, status: "complete" }); + }); + it("accepts explicit zero and omitted total", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ input_tokens: 0, output_tokens: 0 })); + expect(a.snapshot()).toMatchObject({ countedCallCount: 1, totalTokens: 0, status: "complete" }); + }); + it.each([ + { input_tokens: 8 }, + { output_tokens: 3 }, + { total_tokens: 12 }, + { ...full, total_tokens: 20 }, + { input_tokens: 10, output_tokens: "5" }, + { input_tokens: 10, output_tokens: -1 }, + { input_tokens: 10, output_tokens: Infinity }, + { input_tokens: 10, output_tokens: 0.5 }, + ])("retains lower bounds for %j", (usage) => { + const a = createTurnUsageAccumulator(); + a.observe(end(usage)); + expect(a.snapshot()).toMatchObject({ countedCallCount: 1, status: "partial" }); + }); + it("rejects unsafe sums", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ input_tokens: Number.MAX_SAFE_INTEGER, output_tokens: 1 })); + expect(a.snapshot().status).toBe("partial"); + expect(Number.isSafeInteger(a.snapshot().totalTokens)).toBe(true); + }); + it("does not sum unidentified calls and keeps prior totals on failures", () => { + const a = createTurnUsageAccumulator(); + a.observe(end(full)); + a.observe(end(full, "")); + a.observe(start("b")); + a.observe({ event: "on_chat_model_error", run_id: "b" }); + expect(a.snapshot()).toMatchObject({ + totalTokens: 15, + modelCallCount: 2, + countedCallCount: 1, + status: "partial", + }); + }); + it("omits invalid details, includes subsets without adding them", () => { + const a = createTurnUsageAccumulator(); + a.observe( + end({ + ...full, + input_token_details: { cache_read: 4, cache_creation: -1 }, + output_token_details: { reasoning: 2 }, + }), + ); + expect(a.snapshot()).toMatchObject({ + totalTokens: 15, + cachedInputTokens: 4, + reasoningOutputTokens: 2, + status: "complete", + }); + expect(a.snapshot()).not.toHaveProperty("cacheCreationInputTokens"); + }); + it("marks unfinished and missing metadata calls partial", () => { + const a = createTurnUsageAccumulator(); + a.observe(end(full)); + a.observe(start("b")); + expect(a.snapshot().status).toBe("partial"); + a.observe(end(undefined, "b")); + expect(a.snapshot().status).toBe("partial"); + }); + it("ignores raw response metadata and unrelated events", () => { + const a = createTurnUsageAccumulator(); + a.observe({ event: "on_chain_end", run_id: "x" }); + a.observe({ + event: "on_chat_model_end", + run_id: "a", + data: { output: { response_metadata: { usage: full } } }, + }); + expect(a.snapshot()).toMatchObject({ modelCallCount: 1, status: "unavailable" }); + }); + it("prevents overflow across calls", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ input_tokens: Number.MAX_SAFE_INTEGER, output_tokens: 0 })); + a.observe(end(full, "b")); + expect(a.snapshot().status).toBe("partial"); + expect(Number.isSafeInteger(a.snapshot().inputTokens)).toBe(true); + }); +}); diff --git a/electron/ai-edition/deep-agent/usage.ts b/electron/ai-edition/deep-agent/usage.ts new file mode 100644 index 000000000..b5af250da --- /dev/null +++ b/electron/ai-edition/deep-agent/usage.ts @@ -0,0 +1,182 @@ +import type { AgentTurnUsage } from "../../../src/native/contracts"; + +type RecordValue = Record; +function record(value: unknown): RecordValue | undefined { + return value !== null && typeof value === "object" ? (value as RecordValue) : undefined; +} +function count(value: unknown): number | undefined { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; +} +interface CallUsage { + core: { input?: number; output?: number; total?: number }; + inputTokens: number; + outputTokens: number; + totalTokens: number; + complete: boolean; + cachedInputTokens?: number; + cacheCreationInputTokens?: number; + reasoningOutputTokens?: number; +} +function parseUsage(value: unknown): CallUsage | undefined { + const usage = record(value); + if (!usage) return undefined; + const input = count(usage.input_tokens); + const output = count(usage.output_tokens); + const total = count(usage.total_tokens); + if (input === undefined && output === undefined && total === undefined) return undefined; + const sum = (input ?? 0) + (output ?? 0); + const safe = Number.isSafeInteger(sum); + const full = input !== undefined && output !== undefined; + let totalTokens: number; + // Incomplete core fields retain known lower bounds. Full input/output are + // authoritative even when a provider reports an inconsistent total. + if (safe && full) totalTokens = sum; + else if (safe) totalTokens = Math.max(sum, total ?? 0); + else totalTokens = total ?? Math.max(input ?? 0, output ?? 0); + const result: CallUsage = { + core: { input, output, total: total ?? (full && safe ? sum : undefined) }, + inputTokens: input ?? 0, + outputTokens: output ?? 0, + totalTokens, + complete: full && safe && (usage.total_tokens === undefined || total === sum), + }; + const inputDetails = record(usage.input_token_details); + const outputDetails = record(usage.output_token_details); + const cached = count(inputDetails?.cache_read); + const creation = count(inputDetails?.cache_creation); + const reasoning = count(outputDetails?.reasoning); + if (cached !== undefined && input !== undefined && cached <= input) + result.cachedInputTokens = cached; + if (creation !== undefined && input !== undefined && creation <= input) + result.cacheCreationInputTokens = creation; + if (reasoning !== undefined && output !== undefined && reasoning <= output) + result.reasoningOutputTokens = reasoning; + return result; +} + +/** Only completed model messages carry authoritative usage. Stream chunks can + * contain cumulative counters and must never be summed. SDK retries hidden + * inside a model invocation are outside this observable accounting boundary. */ +export function createTurnUsageAccumulator() { + const calls = new Map< + string, + { usage?: CallUsage; ended: boolean; conflict: boolean; failed: boolean } + >(); + let gap = false; + return { + observe(value: unknown) { + const event = record(value); + if ( + !event || + !["on_chat_model_start", "on_chat_model_end", "on_chat_model_error"].includes( + String(event.event), + ) + ) + return; + const id = event.run_id; + if (typeof id !== "string" || !id.trim()) { + gap = true; + return; + } + const call = calls.get(id) ?? { ended: false, conflict: false, failed: false }; + calls.set(id, call); + if (event.event === "on_chat_model_error") call.failed = true; + if (event.event !== "on_chat_model_end") return; + call.ended = true; + const usage = parseUsage(record(record(event.data)?.output)?.usage_metadata); + if (!usage) return; + if (!call.usage) call.usage = usage; + else { + const previous = call.usage; + const core = { ...previous.core }; + let enriched = false; + for (const key of ["input", "output", "total"] as const) { + const value = usage.core[key]; + if (value === undefined) continue; + if (core[key] === undefined) { + core[key] = value; + enriched = true; + } else if (core[key] !== value) call.conflict = true; + } + // Recompute from the union of compatible known fields. A duplicate may + // supply just the missing output, rather than repeating its input count. + const merged = enriched + ? parseUsage({ + input_tokens: core.input, + output_tokens: core.output, + total_tokens: core.total, + })! + : { ...previous }; + for (const key of [ + "cachedInputTokens", + "cacheCreationInputTokens", + "reasoningOutputTokens", + ] as const) { + const first = previous[key]; + const next = usage[key]; + if (first !== undefined && next !== undefined && first !== next) call.conflict = true; + const value = first ?? next; + const bound = key === "reasoningOutputTokens" ? merged.core.output : merged.core.input; + if (value !== undefined && bound !== undefined && value <= bound) merged[key] = value; + else delete merged[key]; + } + call.usage = merged; + } + }, + snapshot(): AgentTurnUsage { + const result: AgentTurnUsage = { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + modelCallCount: calls.size, + countedCallCount: 0, + status: "unavailable", + }; + let complete = !gap && calls.size > 0; + const overflowedDetails = new Set(); + const fields = [ + "inputTokens", + "outputTokens", + "totalTokens", + "cachedInputTokens", + "cacheCreationInputTokens", + "reasoningOutputTokens", + ] as const; + for (const call of calls.values()) { + if (!call.ended || call.failed || call.conflict || !call.usage?.complete) complete = false; + if (!call.usage) continue; + result.countedCallCount++; + for (const field of fields) { + const value = call.usage[field]; + if (value === undefined || overflowedDetails.has(field)) continue; + const sum = (result[field] ?? 0) + value; + if (Number.isSafeInteger(sum)) result[field] = sum; + else { + // Keep the previous safe aggregate as a lower bound on overflow. + complete = false; + if (field !== "inputTokens" && field !== "outputTokens" && field !== "totalTokens") { + delete result[field]; + overflowedDetails.add(field); + } + } + } + } + // Core overflow can retain a smaller lower bound than independently + // summed details. Keep every published subset bounded by that retained core. + for (const field of [ + "cachedInputTokens", + "cacheCreationInputTokens", + "reasoningOutputTokens", + ] as const) { + const bound = field === "reasoningOutputTokens" ? result.outputTokens : result.inputTokens; + if (result[field] !== undefined && result[field] > bound) { + delete result[field]; + complete = false; + } + } + result.status = + result.countedCallCount === 0 ? "unavailable" : complete ? "complete" : "partial"; + return result; + }, + }; +} diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index 46e75ae58..b2e0f09d0 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -194,6 +194,7 @@ function buildChatEventSink(sender: Electron.WebContents, sessionId: string): Ch return { text: (delta) => send({ kind: "text", sessionId, delta }), thinking: (delta) => send({ kind: "thinking", sessionId, delta }), + usage: (usage) => send({ kind: "usage", sessionId, usage }), toolStart: (name, args) => send({ kind: "toolStart", sessionId, name, args }), toolEnd: (name, ok, summary) => send({ kind: "toolEnd", sessionId, name, ok, summary }), error: (message) => send({ kind: "error", sessionId, message }), diff --git a/src/components/ai-edition/ChatUsageSummary.test.tsx b/src/components/ai-edition/ChatUsageSummary.test.tsx new file mode 100644 index 000000000..b9d4099a4 --- /dev/null +++ b/src/components/ai-edition/ChatUsageSummary.test.tsx @@ -0,0 +1,100 @@ +// @vitest-environment jsdom +import "@testing-library/jest-dom"; +import { cleanup, render } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import { TooltipProvider } from "@/components/ui/tooltip"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AgentTurnUsage } from "@/native/contracts"; +import { ChatUsageSummary } from "./ChatUsageSummary"; + +afterEach(cleanup); +const complete: AgentTurnUsage = { + inputTokens: 500, + outputTokens: 160, + totalTokens: 660, + modelCallCount: 3, + countedCallCount: 3, + status: "complete", +}; +const show = (usage?: AgentTurnUsage, durationMs?: number, running = false) => + render( + + + + + , + ); +describe("ChatUsageSummary", () => { + it("shows a concise total with accounting details in the tooltip", () => { + const view = show(complete); + expect(view.getByText("660 tokens")).toHaveAttribute( + "aria-label", + expect.stringContaining("Input: 500"), + ); + expect(view.getByText("660 tokens")).toHaveAttribute( + "aria-label", + expect.stringContaining("3 of 3 model calls counted"), + ); + }); + it("keeps partial accounting details in the tooltip", () => { + const view = show( + { ...complete, totalTokens: 12340, countedCallCount: 2, status: "partial" }, + 45000, + ); + expect(view.getByText("12.3k tokens · 45s")).toHaveAttribute( + "aria-label", + expect.stringContaining("12.3k known tokens · incomplete"), + ); + expect(view.container.textContent).not.toMatch(/known|incomplete/); + }); + it("does not turn unavailable usage into zero", () => { + const view = show({ ...complete, totalTokens: 0, countedCallCount: 0, status: "unavailable" }); + expect(view.container).toBeEmptyDOMElement(); + expect(view.queryByText(/0 tokens/)).toBeNull(); + }); + it("keeps legacy messages quiet", () => { + expect(show().container).toBeEmptyDOMElement(); + }); + it("shows explicitly reported zero", () => { + expect( + show({ ...complete, inputTokens: 0, outputTokens: 0, totalTokens: 0 }).getByText("0 tokens"), + ).toBeInTheDocument(); + }); + it("labels optional details as reported counts and omits missing details", () => { + const view = show({ ...complete, cachedInputTokens: 0, reasoningOutputTokens: 12 }); + const label = view.getByText("660 tokens").getAttribute("aria-label"); + expect(label).toContain("Reported cached input: 0"); + expect(label).toContain("Reported reasoning output: 12"); + expect(label).not.toContain("cache creation"); + }); + it("formats counts for the selected locale", () => { + localStorage.setItem("openscreen-locale", "de"); + const view = show({ ...complete, totalTokens: 12340 }); + expect(view.getByText(/12,3k/)).toBeInTheDocument(); + localStorage.clear(); + }); + it("shows compact totals and elapsed time, with no visible breakdown or model-call count", () => { + const view = show({ ...complete, totalTokens: 12340 }, 45000); + expect(view.getByText("12.3k tokens · 45s")).toBeInTheDocument(); + expect(view.container.textContent).not.toMatch(/Input:|Output:|model calls/); + }); + it("keeps elapsed time on unavailable usage without claiming zero tokens", () => { + const view = show( + { ...complete, status: "unavailable", countedCallCount: 0, totalTokens: 0 }, + 1200, + ); + expect(view.getByText("1s")).toHaveAttribute( + "aria-label", + expect.stringContaining("Token usage unavailable"), + ); + expect(view.container.textContent).toBe("1s"); + }); + it("shows reported running counts as a lower bound and hides unavailable running usage", () => { + const view = show(complete, undefined, true); + expect(view.getByText("≥660 tokens")).toBeInTheDocument(); + view.unmount(); + expect( + show({ ...complete, status: "unavailable" }, undefined, true).container, + ).toBeEmptyDOMElement(); + }); +}); diff --git a/src/components/ai-edition/ChatUsageSummary.tsx b/src/components/ai-edition/ChatUsageSummary.tsx new file mode 100644 index 000000000..013fda122 --- /dev/null +++ b/src/components/ai-edition/ChatUsageSummary.tsx @@ -0,0 +1,79 @@ +import { Tooltip } from "@/components/ui/tooltip"; +import { useI18n, useScopedT } from "@/contexts/I18nContext"; +import type { AgentTurnUsage } from "@/native/contracts"; + +/** Provider-reported usage for this turn, separate from the context-window budget. */ +export function ChatUsageSummary({ + usage, + durationMs, + running = false, +}: { + usage?: AgentTurnUsage; + durationMs?: number; + running?: boolean; +}) { + const { locale } = useI18n(); + const t = useScopedT("editor"); + if (!usage || (running && usage.status === "unavailable")) return null; + const format = (value: number) => new Intl.NumberFormat(locale).format(value); + const tokens = + usage.totalTokens >= 1000 + ? `${new Intl.NumberFormat(locale, { maximumFractionDigits: 1 }).format(usage.totalTokens / 1000)}k` + : format(usage.totalTokens); + const summaryTokens = running ? `≥${tokens}` : tokens; + const summary = + usage.status === "unavailable" ? "" : t("chat.usageComplete", { tokens: summaryTokens }); + const usageDescription = + usage.status === "unavailable" + ? t("chat.usageUnavailable") + : !running && usage.status === "partial" + ? t("chat.usagePartial", { tokens: summaryTokens }) + : t("chat.usageComplete", { tokens: summaryTokens }); + const duration = + !running && durationMs !== undefined + ? new Intl.NumberFormat(locale, { + style: "unit", + unit: "second", + unitDisplay: "narrow", + maximumFractionDigits: 0, + }).format(Math.round(durationMs / 1000)) + : ""; + const line = [summary, duration].filter(Boolean).join(" · "); + if (!line) return null; + const details = [ + [usageDescription, duration].filter(Boolean).join(" · "), + t("chat.usageCoverage", { + counted: format(usage.countedCallCount), + calls: format(usage.modelCallCount), + }), + ]; + if (usage.status !== "unavailable") { + details.push( + t("chat.usageInput", { tokens: format(usage.inputTokens) }), + t("chat.usageOutput", { tokens: format(usage.outputTokens) }), + ); + } + for (const [key, value] of [ + ["usageCachedInput", usage.cachedInputTokens], + ["usageCacheCreation", usage.cacheCreationInputTokens], + ["usageReasoning", usage.reasoningOutputTokens], + ] as const) { + if (value !== undefined) details.push(t(`chat.${key}`, { tokens: format(value) })); + } + const description = details.join(" · "); + return ( + + + {line} + + + ); +} diff --git a/src/components/ai-edition/LeftPanel.tsx b/src/components/ai-edition/LeftPanel.tsx index 144d0b4c7..41ac8a07d 100644 --- a/src/components/ai-edition/LeftPanel.tsx +++ b/src/components/ai-edition/LeftPanel.tsx @@ -14,6 +14,7 @@ import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus"; import { nativeBridgeClient } from "@/native/client"; import type { + AgentTurnUsage, AiEditionChatEvent, AiEditionLlmConfig, AiEditionToolCallSummary, @@ -24,6 +25,7 @@ import { PROVIDER_DEFINITIONS, type ReasoningEffort, } from "../../../electron/ai-edition/provider-registry"; +import { ChatUsageSummary } from "./ChatUsageSummary"; import { ChatWelcome } from "./ChatWelcome"; import { canSendChat } from "./chatAvailability"; import { adoptStoredUserMessage } from "./chatTranscript"; @@ -38,6 +40,8 @@ interface ChatDisplayMessage { /** ISO timestamp, as the main process stores it; formatted only where it is drawn (#1134). */ time?: string; toolCalls?: AiEditionToolCallSummary[]; + usage?: AgentTurnUsage; + durationMs?: number; // ponytail: axcut parity — non-null on user messages that have a // rewind-able document snapshot, so the per-message ↩ button shows. checkpointId?: string | null; @@ -438,6 +442,7 @@ export function ChatStripPanel() { const [messages, setMessages] = useState([]); const [input, setInput] = useState(""); const [busy, setBusy] = useState(false); + const [liveUsage, setLiveUsage] = useState(); const [llmConfig, setLlmConfig] = useState(null); // The dialog itself is mounted in App.tsx so the app menu can reach it from every mode // (issue #420); this panel only asks for it to be opened, and watches it close. @@ -450,6 +455,7 @@ export function ChatStripPanel() { Array<{ id: string; title: string; messageCount: number; createdAt: string }> >([]); const [activeSessionId, setActiveSessionId] = useState(null); + const [sessionReloadVersion, setSessionReloadVersion] = useState(0); // Mirror in a ref so refreshSessions can read the current selection without // listing activeSessionId in its deps — otherwise refreshSessions is recreated // on every selection change, which re-runs the project effect below (with @@ -458,6 +464,19 @@ export function ChatStripPanel() { // freshly-created empty one. const activeSessionIdRef = useRef(activeSessionId); activeSessionIdRef.current = activeSessionId; + const projectIdRef = useRef(projectId); + const viewVersionRef = useRef(0); + const transcriptVersionRef = useRef(0); + const sessionListRequestRef = useRef(0); + const activeTurnRef = useRef<{ + projectId: string; + sessionId: string | null; + viewVersion: number; + } | null>(null); + if (projectIdRef.current !== projectId) { + projectIdRef.current = projectId; + viewVersionRef.current += 1; + } const scrollRef = useRef(null); // ponytail: live reasoning trace for the in-flight turn. Reset at send() // start; deltas append through the chat-event subscription; once the run @@ -496,8 +515,18 @@ export function ChatStripPanel() { }, []); const refreshSessions = useCallback(async (pid: string, preferFirst = false) => { + const request = ++sessionListRequestRef.current; + const viewVersion = viewVersionRef.current; + const transcriptVersion = transcriptVersionRef.current; try { const list = await nativeBridgeClient.aiEdition.chatListSessions(pid); + if ( + projectIdRef.current !== pid || + request !== sessionListRequestRef.current || + viewVersion !== viewVersionRef.current || + transcriptVersion !== transcriptVersionRef.current + ) + return; setSessions(list); if (list.length === 0) { setActiveSessionId(null); @@ -522,12 +551,16 @@ export function ChatStripPanel() { // ponytail: subscribe to streamed chat events so the reasoning trace (and // any future streaming text deltas) lands live instead of arriving all at - // once when chatRun resolves. We only act on `thinking` here — text deltas + // once when chatRun resolves. Usage updates report completed model calls; text deltas // are ignored in the renderer today because the chat already renders the // final assistant text on chatRun resolve, and a parallel live stream // would race the final message. Add `text` handling when that flow lands. useEffect(() => { const unsubChatEvent = window.electronAPI.onAiEditionChatEvent((event: AiEditionChatEvent) => { + if (event.kind === "usage") { + if (event.sessionId === activeTurnRef.current?.sessionId) setLiveUsage(event.usage); + return; + } if (event.kind !== "thinking") return; if (event.sessionId !== thinkingRunSessionRef.current) return; setThinkingText((prev) => prev + event.delta); @@ -536,6 +569,12 @@ export function ChatStripPanel() { }, []); useEffect(() => { + setSessions([]); + setActiveSessionId(null); + activeSessionIdRef.current = null; + setMessages([]); + setThinkingText(""); + thinkingRunSessionRef.current = null; if (!projectId) { setSessions([]); setActiveSessionId(null); @@ -545,17 +584,37 @@ export function ChatStripPanel() { void refreshSessions(projectId, true); }, [projectId, refreshSessions]); + // biome-ignore lint/correctness/useExhaustiveDependencies: completion explicitly reloads a session revisited while its turn was running. useEffect(() => { if (!projectId || !activeSessionId) { setMessages([]); return; } + let cancelled = false; + const viewVersion = viewVersionRef.current; + const transcriptVersion = transcriptVersionRef.current; + const turn = activeTurnRef.current; + // A newly created session is still empty on disk while its first turn runs. + if ( + turn?.projectId === projectId && + turn.sessionId === activeSessionId && + turn.viewVersion === viewVersion + ) + return; void (async () => { try { const session = await nativeBridgeClient.aiEdition.chatSelectSession( projectId, activeSessionId, ); + if ( + cancelled || + projectIdRef.current !== projectId || + activeSessionIdRef.current !== activeSessionId || + viewVersion !== viewVersionRef.current || + transcriptVersion !== transcriptVersionRef.current + ) + return; if (session) { setMessages( session.messages.map((m) => ({ @@ -564,6 +623,8 @@ export function ChatStripPanel() { content: m.content, time: m.createdAt, toolCalls: m.toolCalls, + usage: m.usage, + durationMs: m.durationMs, checkpointId: m.checkpointId ?? null, })), ); @@ -574,7 +635,10 @@ export function ChatStripPanel() { // ponytail: silent — shim mode } })(); - }, [projectId, activeSessionId]); + return () => { + cancelled = true; + }; + }, [projectId, activeSessionId, sessionReloadVersion]); useEffect(() => { scrollRef.current?.scrollTo({ top: scrollRef.current.scrollHeight, behavior: "smooth" }); @@ -591,8 +655,16 @@ export function ChatStripPanel() { openProviderSettings(); return; } + const turn = { projectId, sessionId: activeSessionId, viewVersion: viewVersionRef.current }; + activeTurnRef.current = turn; + const isCurrentView = () => + projectIdRef.current === projectId && + viewVersionRef.current === turn.viewVersion && + activeSessionIdRef.current === turn.sessionId; + transcriptVersionRef.current += 1; setInput(""); setBusy(true); + setLiveUsage(undefined); // ponytail: prepare the live reasoning-trace accumulator. Late events // from a previous run (or from another panel/window) won't match this // sessionId and are dropped by the subscription above. @@ -614,39 +686,42 @@ export function ChatStripPanel() { }, ]); try { - // Mirror axcut's `getOrCreateSession`: the composer works with zero - // setup, so the first message on a project with no sessions yet - // silently starts one instead of no-op'ing. - let sessionId = activeSessionId; - if (!sessionId) { - const created = await nativeBridgeClient.aiEdition.chatCreateSession(projectId); - sessionId = created.id; - setSessions((prev) => [...prev, created]); - setActiveSessionId(sessionId); - } - // ponytail: start collecting the live reasoning trace for THIS run — - // the subscription only appends deltas whose sessionId matches. - thinkingRunSessionRef.current = sessionId; - // Send the current document snapshot so the agent can run edit tools - // against it (P1). Falls back to text-only chat when no doc is open. - // `runAgentTurn` reads the document AND the revision it is at from one snapshot - // before the turn starts, so a manual edit landing while the agent works is - // detectable when its answer comes back. - const { result, applyDocument } = await runAgentTurn((documentSnapshot) => - nativeBridgeClient.aiEdition.chatRun(projectId, sessionId, text, documentSnapshot), - ); - setMessages((prev) => adoptStoredUserMessage(prev, optimisticUserId, result)); - // A rewind confirmation opened during the turn names the message too. Stored without a - // checkpoint, the message has nothing to rewind to and the confirmation closes. - const storedId = result.userMessageId; - if (storedId) { - setRewindFor((open) => - open?.messageId !== optimisticUserId - ? open - : result.userMessageCheckpointId - ? { ...open, messageId: storedId } - : null, - ); + // Capture the document and revision before session creation can yield to a project switch. + const { result, applyDocument } = await runAgentTurn(async (documentSnapshot) => { + // Mirror axcut's `getOrCreateSession`: the composer works with zero + // setup, so the first message on a project with no sessions yet + // silently starts one instead of no-op'ing. + let sessionId = activeSessionId; + if (!sessionId) { + const created = await nativeBridgeClient.aiEdition.chatCreateSession(projectId); + sessionId = created.id; + const wasCurrentView = isCurrentView(); + turn.sessionId = sessionId; + if (wasCurrentView) { + activeSessionIdRef.current = sessionId; + setSessions((prev) => [...prev, created]); + setActiveSessionId(sessionId); + } + } + // ponytail: start collecting the live reasoning trace for THIS run — + // the subscription only appends deltas whose sessionId matches. + if (isCurrentView()) thinkingRunSessionRef.current = sessionId; + return nativeBridgeClient.aiEdition.chatRun(projectId, sessionId, text, documentSnapshot); + }); + if (isCurrentView()) { + setMessages((prev) => adoptStoredUserMessage(prev, optimisticUserId, result)); + // A rewind confirmation opened during the turn names the message too. Stored without a + // checkpoint, the message has nothing to rewind to and the confirmation closes. + const storedId = result.userMessageId; + if (storedId) { + setRewindFor((open) => + open?.messageId !== optimisticUserId + ? open + : result.userMessageCheckpointId + ? { ...open, messageId: storedId } + : null, + ); + } } const assistant = result.assistantMessage; if (result.success && assistant) { @@ -688,29 +763,60 @@ export function ChatStripPanel() { }); } } - setMessages((prev) => [ - ...prev, - { - role: "assistant", - content: assistant.content, - time: new Date().toISOString(), - toolCalls: assistant.toolCalls, - // ponytail: snapshot the live reasoning trace onto the - // finished message so it can be revisited (collapsed by - // default, click-to-expand) instead of vanishing. The - // live accumulator is cleared in `finally`. - thinking: thinkingText || undefined, - }, - ]); - void refreshSessions(projectId); + if (isCurrentView()) + setMessages((prev) => [ + ...prev, + { + id: assistant.id, + usage: assistant.usage, + durationMs: assistant.durationMs, + role: "assistant", + content: assistant.content, + time: assistant.createdAt, + toolCalls: assistant.toolCalls, + // ponytail: snapshot the live reasoning trace onto the + // finished message so it can be revisited (collapsed by + // default, click-to-expand) instead of vanishing. The + // live accumulator is cleared in `finally`. + thinking: thinkingText || undefined, + }, + ]); + if (isCurrentView()) void refreshSessions(projectId); } else { + if (isCurrentView() && result.userMessage) { + const userMessage = result.userMessage; + setMessages((prev) => + prev.map((message) => + message.id === optimisticUserId || message.id === userMessage.id + ? { + ...message, + id: userMessage.id, + checkpointId: userMessage.checkpointId ?? null, + usage: userMessage.usage, + durationMs: userMessage.durationMs, + } + : message, + ), + ); + } toast.error(result.error ?? t("chat.chatFailed")); } + // Returning to the origin while it runs loads an unfinished transcript. + // Reload the persisted result instead of appending to that uncertain snapshot. + if ( + !isCurrentView() && + projectIdRef.current === projectId && + activeSessionIdRef.current === turn.sessionId + ) { + transcriptVersionRef.current += 1; + setSessionReloadVersion((version) => version + 1); + } } catch (err) { toast.error(t("chat.chatFailed"), { description: err instanceof Error ? err.message : String(err), }); } finally { + activeTurnRef.current = null; setBusy(false); // ponytail: stop accepting thinking deltas and drop the in-flight // preview — the snapshot was either attached to the assistant @@ -749,6 +855,7 @@ export function ChatStripPanel() { const confirmRewind = useCallback( async (messageId: string) => { if (!projectId || !activeSessionId) return; + const viewVersion = viewVersionRef.current; try { const result = await nativeBridgeClient.aiEdition.chatRewind( projectId, @@ -763,7 +870,14 @@ export function ChatStripPanel() { const doc = (result as { document?: unknown }).document; // No `expectedRevision`: a rewind REPLACES whatever is live, which is what the // confirmation dialog now says out loud -- including any manual edit made since. - if (doc) await applyAgentDocumentIfCurrent(doc); + if (doc && projectIdRef.current === projectId) await applyAgentDocumentIfCurrent(doc); + if ( + projectIdRef.current !== projectId || + activeSessionIdRef.current !== activeSessionId || + viewVersionRef.current !== viewVersion + ) + return; + transcriptVersionRef.current += 1; setMessages( result.messages.map((m) => ({ id: m.id, @@ -771,6 +885,8 @@ export function ChatStripPanel() { content: m.content, time: m.createdAt, toolCalls: m.toolCalls, + usage: m.usage, + durationMs: m.durationMs, checkpointId: m.checkpointId ?? null, })), ); @@ -902,8 +1018,15 @@ export function ChatStripPanel() { const compactNow = useCallback(async () => { if (!projectId || !activeSessionId || compactNowPending) return; setCompactNowPending(true); + const viewVersion = viewVersionRef.current; try { const result = await nativeBridgeClient.aiEdition.chatCompact(projectId, activeSessionId); + if ( + projectIdRef.current !== projectId || + activeSessionIdRef.current !== activeSessionId || + viewVersionRef.current !== viewVersion + ) + return; if (!result) { toast.info(t("chat.notEnoughHistory")); return; @@ -915,6 +1038,8 @@ export function ChatStripPanel() { content: m.content, time: m.createdAt, toolCalls: m.toolCalls, + usage: m.usage, + durationMs: m.durationMs, checkpointId: m.checkpointId ?? null, })), ); @@ -931,7 +1056,17 @@ export function ChatStripPanel() { const newChat = useCallback(async () => { if (!projectId) return; try { + const viewVersion = viewVersionRef.current; + const transcriptVersion = transcriptVersionRef.current; const created = await nativeBridgeClient.aiEdition.chatCreateSession(projectId); + if ( + projectIdRef.current !== projectId || + viewVersionRef.current !== viewVersion || + transcriptVersionRef.current !== transcriptVersion + ) + return; + viewVersionRef.current += 1; + activeSessionIdRef.current = created.id; setSessions((prev) => [...prev, created]); setActiveSessionId(created.id); setMessages([]); @@ -943,6 +1078,12 @@ export function ChatStripPanel() { }, [projectId, t]); const selectSession = useCallback((id: string) => { + if (activeSessionIdRef.current === id) return; + viewVersionRef.current += 1; + activeSessionIdRef.current = id; + thinkingRunSessionRef.current = null; + setThinkingText(""); + setMessages([]); setActiveSessionId(id); }, []); @@ -1325,6 +1466,7 @@ export function ChatStripPanel() { justifyContent: "flex-end", }} > + {m.role === "user" && m.checkpointId ? ( // The name stays "Rewind to this message"; the tip adds what it does, which // only the confirmation dialog said before. @@ -1415,50 +1557,32 @@ export function ChatStripPanel() { ) : null} ))} - {busy ? ( + {busy && + activeTurnRef.current?.projectId === projectId && + activeTurnRef.current.sessionId === activeSessionId ? (
{t("chat.authorAssistant")}
+
+ + {!thinkingText ? ( + + {t("chat.thinking")} + + ) : null} + +
{thinkingText ? ( -
- -
- setThinkingExpanded((v) => !v)} - label={t("chat.thinking")} - /> -
-
- ) : ( -
- - {t("chat.thinking")} -
- )} + setThinkingExpanded((v) => !v)} + label={t("chat.thinking")} + /> + ) : null}
) : null} diff --git a/src/components/ai-edition/LeftPanel.usage.test.tsx b/src/components/ai-edition/LeftPanel.usage.test.tsx new file mode 100644 index 000000000..7c8c25f33 --- /dev/null +++ b/src/components/ai-edition/LeftPanel.usage.test.tsx @@ -0,0 +1,552 @@ +// @vitest-environment jsdom +import "@testing-library/jest-dom"; +import { act, cleanup, fireEvent, render, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { + AiEditionChatEvent, + AiEditionChatMessage, + AiEditionChatResult, + AiEditionChatSession, + AiEditionChatSessionSummary, +} from "@/native/contracts"; + +const mocks = vi.hoisted(() => ({ + list: vi.fn(), + select: vi.fn(), + run: vi.fn(), + create: vi.fn(), + compact: vi.fn(), + rewind: vi.fn(), +})); +vi.mock("@/native/client", () => ({ + nativeBridgeClient: { + aiEdition: { + llmGetSnapshot: () => + Promise.resolve({ + config: { provider: "openai", model: "gpt-4o" }, + connectedProviders: ["openai"], + }), + chatListSessions: (...args: unknown[]) => mocks.list(...args), + chatSelectSession: (...args: unknown[]) => mocks.select(...args), + chatRun: (...args: unknown[]) => mocks.run(...args), + chatCreateSession: (...args: unknown[]) => mocks.create(...args), + chatBudget: () => Promise.resolve(null), + chatCompact: (...args: unknown[]) => mocks.compact(...args), + chatRewind: (...args: unknown[]) => mocks.rewind(...args), + }, + }, +})); +vi.mock("./Modals", () => ({ + ChatHistoryModal: ({ onSelect }: { onSelect: (id: string) => void }) => ( + <> + + + + ), +})); +vi.mock("@/contexts/I18nContext", async (original) => { + const actual = await original(); + return { + ...actual, + useI18n: () => ({ locale: "en" }), + useScopedT: () => (key: string, vars?: Record) => + key.startsWith("chat.usage") ? `${key} ${JSON.stringify(vars ?? {})}` : key, + }; +}); +const usage = { + inputTokens: 500, + outputTokens: 160, + totalTokens: 660, + modelCallCount: 3, + countedCallCount: 3, + status: "complete" as const, +}; +const message = (id: string, content: string): AiEditionChatMessage => ({ + id, + role: "assistant", + content, + createdAt: "now", + usage, +}); +const summary = (id: string): AiEditionChatSessionSummary => ({ + id, + projectId: "P", + title: id, + messageCount: 1, + createdAt: "now", +}); +const session = ( + id: string, + messages: AiEditionChatMessage[] = [message(id, `History ${id}`)], +): AiEditionChatSession => ({ ...summary(id), projectId: "P", messages }); +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} +let chatEvent: (event: AiEditionChatEvent) => void; +let store: typeof import("@/lib/ai-edition/store/projectStore").useProjectStore; +beforeEach(async () => { + vi.clearAllMocks(); + mocks.list.mockResolvedValue([summary("A"), summary("B")]); + mocks.select.mockImplementation((_pid: string, id: string) => Promise.resolve(session(id))); + mocks.create.mockResolvedValue(summary("A")); + Element.prototype.scrollTo = () => { + /* jsdom does not scroll */ + }; + (window as unknown as { electronAPI: unknown }).electronAPI = { + onAiEditionChatEvent: (listener: (event: AiEditionChatEvent) => void) => { + chatEvent = listener; + return () => { + /* listener belongs to this test render */ + }; + }, + }; + store = (await import("@/lib/ai-edition/store/projectStore")).useProjectStore; + store.setState({ projectId: "P", document: null }); +}); +afterEach(cleanup); +async function panel() { + const { ChatStripPanel } = await import("./LeftPanel"); + const { EditorDialogsProvider } = await import("@/contexts/EditorDialogsContext"); + const { TooltipProvider } = await import("@/components/ui/tooltip"); + return render( + + + + + , + ); +} +async function send(view: ReturnType) { + await waitFor(() => expect(view.getByRole("textbox")).toBeEnabled()); + fireEvent.change(view.getByRole("textbox"), { target: { value: "My prompt" } }); + fireEvent.keyDown(view.getByRole("textbox"), { key: "Enter" }); + await waitFor(() => expect(mocks.run).toHaveBeenCalled()); +} +const usageNode = (view: ReturnType) => view.queryAllByText(/chat.usageComplete/); +describe("chat usage and asynchronous identity", () => { + it("keeps one Thinking label while reasoning streams, alongside live usage", async () => { + const pending = deferred(); + mocks.run.mockReturnValue(pending.promise); + const view = await panel(); + await view.findByText("History A"); + await send(view); + expect(view.getAllByText("chat.thinking")).toHaveLength(1); + act(() => { + chatEvent({ kind: "thinking", sessionId: "A", delta: "Inspecting the project" }); + chatEvent({ kind: "usage", sessionId: "A", usage }); + }); + expect(view.getAllByText("chat.thinking")).toHaveLength(1); + expect(view.getByText(/≥660/)).toBeInTheDocument(); + const thinking = view.getByRole("button", { name: /chat.thinking/ }); + fireEvent.click(thinking); + expect(thinking).toHaveAttribute("aria-expanded", "true"); + await act(async () => + pending.resolve({ success: true, assistantMessage: message("answer", "Done") }), + ); + }); + it("updates live usage beside the activity indicator and replaces it with the completed summary", async () => { + const pending = deferred(); + mocks.run.mockReturnValue(pending.promise); + const view = await panel(); + await view.findByText("History A"); + await send(view); + act(() => chatEvent({ kind: "usage", sessionId: "B", usage })); + expect(view.queryByText(/≥660/)).toBeNull(); + act(() => chatEvent({ kind: "usage", sessionId: "A", usage })); + expect(view.getByText(/≥660/).parentElement?.textContent).toContain("chat.thinking"); + act(() => chatEvent({ kind: "usage", sessionId: "A", usage: { ...usage, totalTokens: 900 } })); + expect(view.getByText(/≥900/)).toBeInTheDocument(); + await act(async () => + pending.resolve({ + success: true, + assistantMessage: { ...message("answer", "Done"), durationMs: 45000 }, + }), + ); + await view.findByText("Done"); + expect(view.queryByText(/≥900/)).toBeNull(); + expect(view.getByText(/45s/)).toBeInTheDocument(); + }); + it("keeps a running count on its original session when the user switches away and back", async () => { + const pending = deferred(); + mocks.run.mockReturnValue(pending.promise); + const view = await panel(); + await view.findByText("History A"); + await send(view); + act(() => chatEvent({ kind: "usage", sessionId: "A", usage })); + fireEvent.click(view.getByText("Select B")); + await view.findByText("History B"); + expect(view.queryByText(/≥660/)).toBeNull(); + act(() => chatEvent({ kind: "usage", sessionId: "A", usage: { ...usage, totalTokens: 900 } })); + expect(view.queryByText(/≥900/)).toBeNull(); + fireEvent.click(view.getByText("Select A")); + await view.findByText("History A"); + expect(view.getByText(/≥900/)).toBeInTheDocument(); + await act(async () => + pending.resolve({ success: true, assistantMessage: message("answer", "Done") }), + ); + }); + + it("loads persisted usage", async () => { + const view = await panel(); + await view.findByText("History A"); + expect(usageNode(view)).toHaveLength(1); + }); + it.each([ + false, + true, + ])("keeps the current session when selected again (pending=%s)", async (pendingTurn) => { + const running = deferred(); + mocks.run.mockReturnValue(running.promise); + const view = await panel(); + await view.findByText("History A"); + if (pendingTurn) await send(view); + fireEvent.click(view.getByText("Select A")); + expect(view.getByText("History A")).toBeInTheDocument(); + if (pendingTurn) { + expect(view.getByText("My prompt")).toBeInTheDocument(); + await act(async () => + running.resolve({ success: true, assistantMessage: message("native-answer", "Answer A") }), + ); + await view.findByText("Answer A"); + expect(usageNode(view)).toHaveLength(2); + } else expect(usageNode(view)).toHaveLength(1); + }); + it("does not let pending new-chat creation erase a subsequently sent turn", async () => { + const creating = deferred(); + const running = deferred(); + mocks.create.mockReturnValue(creating.promise); + mocks.run.mockReturnValue(running.promise); + const view = await panel(); + await view.findByText("History A"); + fireEvent.click(view.getByRole("button", { name: "chat.newConversation" })); + await waitFor(() => expect(mocks.create).toHaveBeenCalled()); + await send(view); + await act(async () => creating.resolve(summary("B"))); + expect(view.getByText("History A")).toBeInTheDocument(); + expect(view.getByText("My prompt")).toBeInTheDocument(); + await act(async () => + running.resolve({ success: true, assistantMessage: message("native-answer", "Answer A") }), + ); + await view.findByText("Answer A"); + expect(usageNode(view)).toHaveLength(2); + }); + + it.each([ + true, + false, + ])("does not put a switched session's completed turn in the current session (success=%s)", async (success) => { + const pending = deferred(); + mocks.run.mockReturnValue(pending.promise); + const view = await panel(); + await view.findByText("History A"); + await send(view); + fireEvent.click(view.getByText("Select B")); + await view.findByText("History B"); + await act(async () => + pending.resolve( + success + ? { success: true, assistantMessage: message("native-answer", "Answer A") } + : { + success: false, + error: "failed", + userMessage: { ...message("native-user", "My prompt"), role: "user" }, + }, + ), + ); + expect(view.queryByText("Answer A")).toBeNull(); + expect(view.queryByText("My prompt")).toBeNull(); + mocks.select.mockImplementation((_pid: string, id: string) => + Promise.resolve(session(id, [message("persisted", "Persisted A")])), + ); + fireEvent.click(view.getByText("Select A")); + await view.findByText("Persisted A"); + expect(usageNode(view)).toHaveLength(1); + }); + it.each([ + [true, false], + [true, true], + [false, false], + [false, true], + ])("reloads a returned session when its turn completes (success=%s, old load late=%s)", async (success, oldLoadLate) => { + const running = deferred(); + const returnLoading = deferred(); + const persistedUser: AiEditionChatMessage = { + ...message("native-user", "My prompt"), + role: "user", + checkpointId: "native-user", + usage: success ? undefined : usage, + }; + let completed = false; + let aLoads = 0; + mocks.select.mockImplementation((_pid: string, id: string) => { + if (id === "B") return Promise.resolve(session("B")); + aLoads += 1; + if (completed) + return Promise.resolve( + session( + "A", + success ? [persistedUser, message("native-answer", "Answer A")] : [persistedUser], + ), + ); + if (aLoads > 1) return returnLoading.promise; + return Promise.resolve(session("A")); + }); + mocks.run.mockReturnValue(running.promise); + const view = await panel(); + await view.findByText("History A"); + await send(view); + fireEvent.click(view.getByText("Select B")); + await view.findByText("History B"); + fireEvent.click(view.getByText("Select A")); + await waitFor(() => expect(aLoads).toBe(2)); + if (!oldLoadLate) await act(async () => returnLoading.resolve(session("A", [persistedUser]))); + await act(async () => { + completed = true; + running.resolve( + success + ? { success: true, assistantMessage: message("native-answer", "Answer A") } + : { success: false, userMessage: persistedUser }, + ); + }); + await waitFor(() => expect(aLoads).toBe(3)); + if (oldLoadLate) await act(async () => returnLoading.resolve(session("A", [persistedUser]))); + expect(view.getAllByText("My prompt")).toHaveLength(1); + expect(usageNode(view)).toHaveLength(1); + if (success) expect(view.getAllByText("Answer A")).toHaveLength(1); + else { + expect(view.queryByText("Answer A")).toBeNull(); + mocks.rewind.mockResolvedValue({ success: true, messages: [], prompt: "My prompt" }); + fireEvent.click(view.getByRole("button", { name: "chat.rewindToMessage" })); + fireEvent.click(await view.findByRole("button", { name: "chat.rewindConfirm" })); + await waitFor(() => expect(mocks.rewind).toHaveBeenCalledWith("P", "A", "native-user")); + } + }); + it.each([ + true, + false, + ])("reloads a first session created while its project was away (success=%s)", async (success) => { + const creating = deferred(); + const running = deferred(); + let created = false; + let completed = false; + const persistedUser: AiEditionChatMessage = { + ...message("native-user", "My prompt"), + role: "user", + checkpointId: "native-user", + usage: success ? undefined : usage, + }; + mocks.list.mockImplementation((pid: string) => + Promise.resolve(pid === "Q" ? [summary("B")] : created ? [summary("A")] : []), + ); + mocks.select.mockImplementation((_pid: string, id: string) => + Promise.resolve( + session( + id, + id === "B" + ? [message("B", "History B")] + : completed + ? success + ? [persistedUser, message("native-answer", "Answer A")] + : [persistedUser] + : [], + ), + ), + ); + mocks.create.mockReturnValue(creating.promise); + mocks.run.mockReturnValue(running.promise); + const view = await panel(); + await waitFor(() => expect(mocks.list).toHaveBeenCalledWith("P")); + await waitFor(() => expect(view.getByRole("textbox")).toBeEnabled()); + fireEvent.change(view.getByRole("textbox"), { target: { value: "My prompt" } }); + fireEvent.keyDown(view.getByRole("textbox"), { key: "Enter" }); + await waitFor(() => expect(mocks.create).toHaveBeenCalled()); + await act(async () => store.setState({ projectId: "Q" })); + await view.findByText("History B"); + await act(async () => { + created = true; + creating.resolve(summary("A")); + }); + await waitFor(() => expect(mocks.run).toHaveBeenCalled()); + await act(async () => store.setState({ projectId: "P" })); + await waitFor(() => expect(mocks.select).toHaveBeenCalledWith("P", "A")); + await act(async () => { + completed = true; + running.resolve( + success + ? { success: true, assistantMessage: message("native-answer", "Answer A") } + : { success: false, userMessage: persistedUser }, + ); + }); + await view.findByText("My prompt"); + expect(usageNode(view)).toHaveLength(1); + if (success) expect(view.getAllByText("Answer A")).toHaveLength(1); + else { + mocks.rewind.mockResolvedValue({ success: true, messages: [], prompt: "My prompt" }); + fireEvent.click(view.getByRole("button", { name: "chat.rewindToMessage" })); + fireEvent.click(await view.findByRole("button", { name: "chat.rewindConfirm" })); + await waitFor(() => expect(mocks.rewind).toHaveBeenCalledWith("P", "A", "native-user")); + } + }); + + it("reloads the original project after leaving and returning before completion", async () => { + const running = deferred(); + let completed = false; + mocks.list.mockImplementation((pid: string) => + Promise.resolve([summary(pid === "Q" ? "B" : "A")]), + ); + mocks.select.mockImplementation((_pid: string, id: string) => + Promise.resolve( + session( + id, + completed && id === "A" + ? [message("native-answer", "Answer A")] + : [message(id, `History ${id}`)], + ), + ), + ); + mocks.run.mockReturnValue(running.promise); + const view = await panel(); + await view.findByText("History A"); + await send(view); + await act(async () => store.setState({ projectId: "Q" })); + await view.findByText("History B"); + await act(async () => store.setState({ projectId: "P" })); + await view.findByText("History A"); + await act(async () => { + completed = true; + running.resolve({ success: true, assistantMessage: message("native-answer", "Answer A") }); + }); + await view.findByText("Answer A"); + expect(usageNode(view)).toHaveLength(1); + }); + + it("ignores a reversed session load", async () => { + const late = deferred(); + mocks.select.mockImplementation((_pid: string, id: string) => + id === "A" ? late.promise : Promise.resolve(session(id)), + ); + const view = await panel(); + await waitFor(() => expect(mocks.select).toHaveBeenCalled()); + fireEvent.click(view.getByText("Select B")); + await view.findByText("History B"); + await act(async () => late.resolve(session("A"))); + expect(view.queryByText("History A")).toBeNull(); + }); + it("keeps the optimistic message and reply when initial loading finishes late", async () => { + const late = deferred(); + mocks.list.mockResolvedValue([]); + mocks.create.mockImplementation(async () => { + mocks.list.mockResolvedValue([summary("A")]); + return summary("A"); + }); + mocks.select.mockReturnValue(late.promise); + mocks.run.mockResolvedValue({ + success: true, + assistantMessage: message("native-answer", "Answer A"), + }); + const view = await panel(); + await waitFor(() => expect(mocks.list).toHaveBeenCalled()); + await send(view); + await view.findByText("Answer A"); + await act(async () => late.resolve(session("A", []))); + expect(view.getByText("My prompt")).toBeInTheDocument(); + expect(view.getByText("Answer A")).toBeInTheDocument(); + }); + it("keeps a failed user when an existing session's initial empty load arrives late", async () => { + const loading = deferred(); + const running = deferred(); + mocks.select.mockReturnValue(loading.promise); + mocks.run.mockReturnValue(running.promise); + const view = await panel(); + await waitFor(() => expect(mocks.select).toHaveBeenCalled()); + await send(view); + await act(async () => loading.resolve(session("A", []))); + expect(view.getByText("My prompt")).toBeInTheDocument(); + await act(async () => + running.resolve({ + success: false, + userMessage: { ...message("native-user", "My prompt"), role: "user" }, + }), + ); + expect(usageNode(view)).toHaveLength(1); + }); + it("attaches failure usage to the optimistic user without an assistant", async () => { + mocks.run.mockResolvedValue({ + success: false, + userMessage: { ...message("native-user", "My prompt"), role: "user" }, + }); + const view = await panel(); + await view.findByText("History A"); + await send(view); + await waitFor(() => expect(usageNode(view)).toHaveLength(2)); + expect(view.getAllByText("My prompt")).toHaveLength(1); + }); + it.each([ + "compact", + "rewind", + ] as const)("retains usage after %s and ignores a late result after switching", async (operation) => { + mocks.select.mockImplementation((_pid: string, id: string) => + Promise.resolve( + session(id, [{ ...message(id, `History ${id}`), role: "user", checkpointId: id }]), + ), + ); + const response = { + success: true, + prompt: "restored", + messages: [message("restored", "Restored")], + session: session("A", [message("restored", "Restored")]), + }; + mocks[operation].mockResolvedValue(response); + const view = await panel(); + await view.findByText("History A"); + const trigger = async () => { + if (operation === "compact") + fireEvent.click(view.getByRole("button", { name: "chat.compactContext" })); + else { + fireEvent.click(view.getByRole("button", { name: "chat.rewindToMessage" })); + fireEvent.click(await view.findByRole("button", { name: "chat.rewindConfirm" })); + } + }; + await trigger(); + await view.findByText("Restored"); + expect(usageNode(view)).toHaveLength(1); + fireEvent.click(view.getByText("Select B")); + await view.findByText("History B"); + const late = deferred(); + mocks[operation].mockReturnValue(late.promise); + await trigger(); + fireEvent.click(view.getByText("Select A")); + await view.findByText("History A"); + await act(async () => late.resolve(response)); + expect(view.queryByText("Restored")).toBeNull(); + }); + it("ignores stale project lists and run results", async () => { + const lateList = deferred(); + const lateRun = deferred(); + mocks.run.mockReturnValue(lateRun.promise); + const view = await panel(); + await view.findByText("History A"); + await send(view); + mocks.list.mockImplementation((pid: string) => + pid === "Q" ? lateList.promise : Promise.resolve([summary("B")]), + ); + await act(async () => store.setState({ projectId: "Q" })); + await act(async () => store.setState({ projectId: "R" })); + await view.findByText("History B"); + await act(async () => { + lateList.resolve([summary("A")]); + lateRun.resolve({ success: true, assistantMessage: message("answer", "Answer A") }); + }); + expect(view.queryByText("Answer A")).toBeNull(); + expect(view.queryByText("History A")).toBeNull(); + expect(view.getByText("History B")).toBeInTheDocument(); + }); +}); diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index 143f19e57..a11a43856 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -313,7 +313,16 @@ "subtitle": "التبديل بين الجلسات أو إنشاء جلسة جديدة", "empty": "لا توجد محادثات بعد.", "msgsCount": "{{count}} رسالة · {{date}}" - } + }, + "usageComplete": "{{tokens}} رمزًا", + "usagePartial": "{{tokens}} رمزًا معروفًا · غير مكتمل", + "usageUnavailable": "استخدام الرموز غير متاح", + "usageCoverage": "تم احتساب {{counted}} من {{calls}} استدعاءات للنموذج", + "usageInput": "الإدخال: {{tokens}}", + "usageOutput": "الإخراج: {{tokens}}", + "usageCachedInput": "إدخال التخزين المؤقت المُبلّغ عنه: {{tokens}}", + "usageCacheCreation": "إدخال إنشاء التخزين المؤقت المُبلّغ عنه: {{tokens}}", + "usageReasoning": "إخراج الاستدلال المُبلّغ عنه: {{tokens}}" }, "newProjectDialog": { "title": "مشروع جديد", diff --git a/src/i18n/locales/cs/editor.json b/src/i18n/locales/cs/editor.json index 53ba12667..54d7f7215 100644 --- a/src/i18n/locales/cs/editor.json +++ b/src/i18n/locales/cs/editor.json @@ -313,7 +313,16 @@ "subtitle": "Přepínání nebo vytváření relací", "empty": "Zatím žádné konverzace.", "msgsCount": "{{count}} zpráv · {{date}}" - } + }, + "usageComplete": "{{tokens}} tokenů", + "usagePartial": "{{tokens}} známých tokenů · neúplné", + "usageUnavailable": "Využití tokenů není dostupné", + "usageCoverage": "Započítáno {{counted}} z {{calls}} volání modelu", + "usageInput": "Vstup: {{tokens}}", + "usageOutput": "Výstup: {{tokens}}", + "usageCachedInput": "Hlášený vstup z mezipaměti: {{tokens}}", + "usageCacheCreation": "Hlášený vstup pro vytvoření mezipaměti: {{tokens}}", + "usageReasoning": "Hlášený výstup uvažování: {{tokens}}" }, "newProjectDialog": { "title": "Nový projekt", diff --git a/src/i18n/locales/de/editor.json b/src/i18n/locales/de/editor.json index 196e06f51..81e25075e 100644 --- a/src/i18n/locales/de/editor.json +++ b/src/i18n/locales/de/editor.json @@ -313,7 +313,16 @@ "subtitle": "Sitzungen wechseln oder anlegen", "empty": "Noch keine Unterhaltungen.", "msgsCount": "{{count}} Nachr. · {{date}}" - } + }, + "usageComplete": "{{tokens}} Tokens", + "usagePartial": "{{tokens}} bekannte Tokens · unvollständig", + "usageUnavailable": "Tokenverbrauch nicht verfügbar", + "usageCoverage": "{{counted}} von {{calls}} Modellaufrufen erfasst", + "usageInput": "Eingabe: {{tokens}}", + "usageOutput": "Ausgabe: {{tokens}}", + "usageCachedInput": "Gemeldete Cache-Eingabe: {{tokens}}", + "usageCacheCreation": "Gemeldete Eingabe zur Cache-Erstellung: {{tokens}}", + "usageReasoning": "Gemeldete Denkausgabe: {{tokens}}" }, "newProjectDialog": { "title": "Neues Projekt", diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index fb0cb8917..3885590b0 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -313,7 +313,16 @@ "subtitle": "Switch or create sessions", "empty": "No conversations yet.", "msgsCount": "{{count}} msgs · {{date}}" - } + }, + "usageComplete": "{{tokens}} tokens", + "usagePartial": "{{tokens}} known tokens · incomplete", + "usageUnavailable": "Token usage unavailable", + "usageCoverage": "{{counted}} of {{calls}} model calls counted", + "usageInput": "Input: {{tokens}}", + "usageOutput": "Output: {{tokens}}", + "usageCachedInput": "Reported cached input: {{tokens}}", + "usageCacheCreation": "Reported cache creation input: {{tokens}}", + "usageReasoning": "Reported reasoning output: {{tokens}}" }, "newProjectDialog": { "title": "New project", diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 89bf7d390..ec7e7f92f 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -313,7 +313,16 @@ "subtitle": "Cambia o crea sesiones", "empty": "Aún no hay conversaciones.", "msgsCount": "{{count}} mensajes · {{date}}" - } + }, + "usageComplete": "{{tokens}} tokens", + "usagePartial": "{{tokens}} tokens conocidos · incompleto", + "usageUnavailable": "Uso de tokens no disponible", + "usageCoverage": "{{counted}} de {{calls}} llamadas al modelo contabilizadas", + "usageInput": "Entrada: {{tokens}}", + "usageOutput": "Salida: {{tokens}}", + "usageCachedInput": "Entrada en caché declarada: {{tokens}}", + "usageCacheCreation": "Entrada de creación de caché declarada: {{tokens}}", + "usageReasoning": "Salida de razonamiento declarada: {{tokens}}" }, "newProjectDialog": { "title": "Nuevo proyecto", diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 92ec22bcb..4b6c72d0b 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -313,7 +313,16 @@ "subtitle": "Changer ou créer des sessions", "empty": "Aucune conversation pour l'instant.", "msgsCount": "{{count}} messages · {{date}}" - } + }, + "usageComplete": "{{tokens}} tokens", + "usagePartial": "{{tokens}} tokens connus · incomplet", + "usageUnavailable": "Utilisation des tokens indisponible", + "usageCoverage": "{{counted}} appels au modèle comptés sur {{calls}}", + "usageInput": "Entrée : {{tokens}}", + "usageOutput": "Sortie : {{tokens}}", + "usageCachedInput": "Entrée en cache déclarée : {{tokens}}", + "usageCacheCreation": "Entrée de création du cache déclarée : {{tokens}}", + "usageReasoning": "Sortie de raisonnement déclarée : {{tokens}}" }, "newProjectDialog": { "title": "Nouveau projet", diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index e9a61562e..f0386ff02 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -313,7 +313,16 @@ "subtitle": "Cambia o crea sessioni", "empty": "Nessuna conversazione ancora.", "msgsCount": "{{count}} messaggi · {{date}}" - } + }, + "usageComplete": "{{tokens}} token", + "usagePartial": "{{tokens}} token noti · incompleto", + "usageUnavailable": "Utilizzo dei token non disponibile", + "usageCoverage": "{{counted}} di {{calls}} chiamate al modello conteggiate", + "usageInput": "Input: {{tokens}}", + "usageOutput": "Output: {{tokens}}", + "usageCachedInput": "Input in cache segnalato: {{tokens}}", + "usageCacheCreation": "Input di creazione cache segnalato: {{tokens}}", + "usageReasoning": "Output di ragionamento segnalato: {{tokens}}" }, "newProjectDialog": { "title": "Nuovo progetto", diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index 10d8f7aab..a1b1dae26 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -313,7 +313,16 @@ "subtitle": "セッションを切り替えるか新規作成", "empty": "まだ会話はありません。", "msgsCount": "{{count}} 件のメッセージ · {{date}}" - } + }, + "usageComplete": "{{tokens}} トークン", + "usagePartial": "既知の {{tokens}} トークン · 不完全", + "usageUnavailable": "トークン使用量を取得できません", + "usageCoverage": "モデル呼び出し {{calls}} 回中 {{counted}} 回を集計", + "usageInput": "入力:{{tokens}}", + "usageOutput": "出力:{{tokens}}", + "usageCachedInput": "報告されたキャッシュ入力:{{tokens}}", + "usageCacheCreation": "報告されたキャッシュ作成入力:{{tokens}}", + "usageReasoning": "報告された推論出力:{{tokens}}" }, "newProjectDialog": { "title": "新規プロジェクト", diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index fa1dbc148..e66397d0d 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -313,7 +313,16 @@ "subtitle": "세션 전환 또는 새로 만들기", "empty": "아직 대화가 없습니다.", "msgsCount": "메시지 {{count}}개 · {{date}}" - } + }, + "usageComplete": "{{tokens}} 토큰", + "usagePartial": "확인된 {{tokens}} 토큰 · 불완전", + "usageUnavailable": "토큰 사용량을 확인할 수 없음", + "usageCoverage": "모델 호출 {{calls}}회 중 {{counted}}회 집계", + "usageInput": "입력: {{tokens}}", + "usageOutput": "출력: {{tokens}}", + "usageCachedInput": "보고된 캐시 입력: {{tokens}}", + "usageCacheCreation": "보고된 캐시 생성 입력: {{tokens}}", + "usageReasoning": "보고된 추론 출력: {{tokens}}" }, "newProjectDialog": { "title": "새 프로젝트", diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index 8dcaff24a..bc1cd11c4 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -313,7 +313,16 @@ "subtitle": "Alternar ou criar sessões", "empty": "Ainda não há conversas.", "msgsCount": "{{count}} mensagens · {{date}}" - } + }, + "usageComplete": "{{tokens}} tokens", + "usagePartial": "{{tokens}} tokens conhecidos · incompleto", + "usageUnavailable": "Uso de tokens indisponível", + "usageCoverage": "{{counted}} de {{calls}} chamadas ao modelo contabilizadas", + "usageInput": "Entrada: {{tokens}}", + "usageOutput": "Saída: {{tokens}}", + "usageCachedInput": "Entrada em cache informada: {{tokens}}", + "usageCacheCreation": "Entrada de criação de cache informada: {{tokens}}", + "usageReasoning": "Saída de raciocínio informada: {{tokens}}" }, "newProjectDialog": { "title": "Novo projeto", diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index 827488c31..de9acd6b4 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -313,7 +313,16 @@ "subtitle": "Переключить или создать сессию", "empty": "Пока нет бесед.", "msgsCount": "{{count}} сообщ. · {{date}}" - } + }, + "usageComplete": "{{tokens}} токенов", + "usagePartial": "Известно {{tokens}} токенов · неполные данные", + "usageUnavailable": "Расход токенов недоступен", + "usageCoverage": "Учтено {{counted}} из {{calls}} вызовов модели", + "usageInput": "Вход: {{tokens}}", + "usageOutput": "Выход: {{tokens}}", + "usageCachedInput": "Сообщённый кэшированный вход: {{tokens}}", + "usageCacheCreation": "Сообщённый вход создания кэша: {{tokens}}", + "usageReasoning": "Сообщённый выход рассуждений: {{tokens}}" }, "newProjectDialog": { "title": "Новый проект", diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index 78efe59a1..2b862de5d 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -313,7 +313,16 @@ "subtitle": "Oturum değiştir veya oluştur", "empty": "Henüz konuşma yok.", "msgsCount": "{{count}} mesaj · {{date}}" - } + }, + "usageComplete": "{{tokens}} token", + "usagePartial": "Bilinen {{tokens}} token · eksik", + "usageUnavailable": "Token kullanımı mevcut değil", + "usageCoverage": "{{calls}} model çağrısından {{counted}} tanesi sayıldı", + "usageInput": "Girdi: {{tokens}}", + "usageOutput": "Çıktı: {{tokens}}", + "usageCachedInput": "Bildirilen önbellek girdisi: {{tokens}}", + "usageCacheCreation": "Bildirilen önbellek oluşturma girdisi: {{tokens}}", + "usageReasoning": "Bildirilen akıl yürütme çıktısı: {{tokens}}" }, "newProjectDialog": { "title": "Yeni proje", diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 5341e7021..db9cc11b0 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -313,7 +313,16 @@ "subtitle": "Chuyển đổi hoặc tạo phiên", "empty": "Chưa có cuộc trò chuyện nào.", "msgsCount": "{{count}} tin nhắn · {{date}}" - } + }, + "usageComplete": "{{tokens}} token", + "usagePartial": "{{tokens}} token đã biết · chưa đầy đủ", + "usageUnavailable": "Không có dữ liệu sử dụng token", + "usageCoverage": "Đã tính {{counted}} trên {{calls}} lượt gọi mô hình", + "usageInput": "Đầu vào: {{tokens}}", + "usageOutput": "Đầu ra: {{tokens}}", + "usageCachedInput": "Đầu vào bộ nhớ đệm được báo cáo: {{tokens}}", + "usageCacheCreation": "Đầu vào tạo bộ nhớ đệm được báo cáo: {{tokens}}", + "usageReasoning": "Đầu ra suy luận được báo cáo: {{tokens}}" }, "newProjectDialog": { "title": "Dự án mới", diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index a50819a0f..5289f515e 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -313,7 +313,16 @@ "subtitle": "切换或新建会话", "empty": "暂无对话。", "msgsCount": "{{count}} 条消息 · {{date}}" - } + }, + "usageComplete": "{{tokens}} 个 token", + "usagePartial": "已知 {{tokens}} 个 token · 不完整", + "usageUnavailable": "Token 用量不可用", + "usageCoverage": "已统计 {{counted}} / {{calls}} 次模型调用", + "usageInput": "输入:{{tokens}}", + "usageOutput": "输出:{{tokens}}", + "usageCachedInput": "已报告的缓存输入:{{tokens}}", + "usageCacheCreation": "已报告的缓存创建输入:{{tokens}}", + "usageReasoning": "已报告的推理输出:{{tokens}}" }, "newProjectDialog": { "title": "新建项目", diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index f0a7c9703..f28e7f87f 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -313,7 +313,16 @@ "subtitle": "切換或新增工作階段", "empty": "尚無對話。", "msgsCount": "{{count}} 則訊息 · {{date}}" - } + }, + "usageComplete": "{{tokens}} 個 token", + "usagePartial": "已知 {{tokens}} 個 token · 不完整", + "usageUnavailable": "Token 用量無法取得", + "usageCoverage": "已統計 {{counted}} / {{calls}} 次模型呼叫", + "usageInput": "輸入:{{tokens}}", + "usageOutput": "輸出:{{tokens}}", + "usageCachedInput": "已回報的快取輸入:{{tokens}}", + "usageCacheCreation": "已回報的快取建立輸入:{{tokens}}", + "usageReasoning": "已回報的推理輸出:{{tokens}}" }, "newProjectDialog": { "title": "新增專案", diff --git a/src/native/contracts.ts b/src/native/contracts.ts index 521cc993e..23024511f 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -384,7 +384,23 @@ export interface AiEditionToolCallSummary { summary: string; } +/** Usage reported by observable model calls in one agent turn, not HTTP billing. */ +export interface AgentTurnUsage { + inputTokens: number; + outputTokens: number; + totalTokens: number; + modelCallCount: number; + countedCallCount: number; + status: "complete" | "partial" | "unavailable"; + cachedInputTokens?: number; + cacheCreationInputTokens?: number; + reasoningOutputTokens?: number; +} + export interface AiEditionChatMessage { + /** Elapsed agent turn time, measured in the main process. Absent on older messages. */ + durationMs?: number; + usage?: AgentTurnUsage; id: string; role: "user" | "assistant"; content: string; @@ -399,6 +415,7 @@ export interface AiEditionChatMessage { } export interface AiEditionChatResult { + userMessage?: AiEditionChatMessage; success: boolean; assistantMessage?: AiEditionChatMessage; /** Updated document when the agent ran write tools during this turn. */ @@ -1022,6 +1039,7 @@ export type NativeBridgeEventName = export type AiEditionChatEvent = | { kind: "text"; sessionId: string; delta: string } | { kind: "thinking"; sessionId: string; delta: string } + | { kind: "usage"; sessionId: string; usage: AgentTurnUsage } | { kind: "toolStart"; sessionId: string; name: string; args: unknown } | { kind: "toolEnd"; sessionId: string; name: string; ok: boolean; summary?: string } | { kind: "error"; sessionId: string; message: string }; diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 340df1480..d5bce6475 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -913,6 +913,7 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | Date | Build / tag | Platform | Pass/fail | Notes | |------|-------------|----------|-----------|-------| +| 2026-10-13 | dev `codex/agent-turn-token-usage`, based on `9c55596e14b65d08a1a2c81a00cedd70dddd9be1`, uncommitted issue #1096 changes | macOS 26.4, arm64 | Partial — token usage chat slice | Real Electron driven by computer-use OS input, isolated test profile and local scripted OpenAI-compatible provider (no real API key or provider billing). Observed complete 660 tokens / 3 calls, known partial 440 / 2 of 3, failed-turn 110 / 1 of 2 on the original user message, and unavailable usage; inspected the project chat sidecar for matching values and ownership. After a delayed request in Conversation 1, Conversation 2 stayed empty when it completed. Returned to Conversation 1 while a later request was still running and observed its final reply and usage once. A failed request retained 110 known tokens on its original user message. Fully quit and restarted Electron with the same isolated profile, reopened Studio and verified all usage states and reported details restored; Conversation 2 remained empty. Usage is separate from the existing context indicator. Skipped: real paid-provider requests; capture, webcam/audio, compositor/preview/export (chat-only change, native binaries unavailable); Windows/Linux and release promotion. Restart persistence, old v1 messages, deletion/rewind and async UI races are covered separately by automated tests. | | 2026-09-23 | dev `fix/725-hidden-cursor-auto-zoom`; rebuilt compositor addon SHA256 `a0bcd61e17084c010141049a9978121e45be2985449ff1618ab6a3145f8381f6` | Windows 11 Home 26200 | Partial — issue #725 slider slice passed | Human-driven real recording in the Electron dev build: moving a slider whose application cursor becomes hidden kept the reconstructed cursor hidden in the Studio recording and did not create a click-driven automatic zoom for that press. A visible control click still produced the normal automatic zoom. Sidecar inspection independently confirmed the hidden interval (`visible: false`) and that no zoom was centred on the press immediately preceding it. Recording, stop and editor open passed. Not covered: export, packaged build, audio/webcam, tray/HUD, other editor features, macOS or Linux. | | 2026-09-12 | dev `fix/643-gif-export-cancel`, based on `fbe461e9`; rebuilt compositor addon SHA256 `128c89fe65a459b3f17721d610ea9a78e0d3e980e9eb0b5fb2b0e5d018619e15` | Windows | Partial — export cancellation passed | Computer-use OS input, real Electron and native save dialog, imported synthetic 8 s source. GIF 15 FPS / Medium (720p) / loop on: observed 55/120 frames, clicked Cancel while rendering (pending snapshot 104/120), returned to the same options; no final or partial output remained. Retried in that dialog to the same path and received Saved to; resulting GIF is 1280×720, 120 frames, 8,200,202 bytes, fully decodes with ffmpeg (exit 0). Automated real-addon tests separately pass existing-destination preservation, GIF retry, ordinary GIF and MP4 export. Desktop adapter required screenshot-based coordinates and Ctrl+O to reach the editor; HUD reachability is not claimed. An unrelated process error window partly obscured the retry success screenshot; file verification independently passed. Skipped: recording, audio/mic, webcam, tray/HUD, effect/caption comparisons, second viewer, macOS/Linux and release promotion. Task-local profile and evidence retained; test app stopped. | | 2026-07-31 | dev build, `claude/e2e-tests-v1-8-0-474894` (e9578f09) | macOS 26.5, M1 | Partial — 1 defect | Ran launch/HUD, media, modifier anchoring, and export. **Defect: a dangling asset blanks the preview.** Modifier anchoring across a reorder verified in preview and in the exported frames. macOS export produced 1280×720 h264 + AAC at ~2× realtime. Chat sections skipped: no AI provider configured. HUD drag not runnable under computer-use (drop point is the desktop). | @@ -994,3 +995,5 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-17 | PR #1319 (`2236580`), artifact `openscreen-linux` of build run 39046512213, AppImage run directly. Reference: the `v2.1.0` AppImage from the GitHub release | Ubuntu 24.04.4, GNOME 46 Wayland, AMD Radeon 610M (RADV); `[d3d] … -> backend Hardware`, no CPU fallback | **Partial — fix slice only, no defect** | Same MCP pass and project as the rows above, on the hardware Vulkan backend. `orbit` (12 frames) and `left` (4 s, 6 s): the whole card with four rounded corners, no flat cut. `v2.1.0` reproduces the bug (top and bottom edges flat, corners cut from 1.5 s). `none` and the flat frames at 1.0 and 9.5 s are byte-identical to `v2.1.0` (equal md5). **Not covered:** the editor preview; `right`; a full MP4 export. | | 2026-10-17 | `v2.1.1-rc.1`, `release/v2.1.1` @ `6ec38bf` (= `v2.1.0` + #1319), assets of the GitHub pre-release (build run 39069024444). Windows: `Openscreen.Setup.2.1.1-rc.1-x64.exe`, sha512 matched `latest.yml`, unpacked with 7-Zip and run in place, version 2.1.1. macOS: `Openscreen-macOS-Apple-Silicon-2.1.1-rc.1.dmg`, sha256 matched the asset digest; `spctl` accepted, Notarized Developer ID, on the DMG and the app; ticket stapled to the DMG only, as on 2.1.0-rc.6; `CFBundleShortVersionString` 2.1.1-rc.1. Linux: `Openscreen-Linux-2.1.1-rc.1.AppImage`, sha512 matched `latest-linux.yml` | Windows 11 Home 26200, RTX 4070 Ti; macOS 27.0.1 (26A434), Mac mini M1 8 GB; Ubuntu 24.04.4, GNOME 46 Wayland, Radeon 610M, hardware Vulkan backend | **Partial — patch pass toward stable: no defect on any of the three** | Scoped to what the patch changes (the compositor geometry of a tilted filled card), in MCP over stdio with an isolated `--user-data-dir`, on the project of the PR rows above. On all three: `orbit` (12 frames) and `left` show the whole card with four rounded corners and no flat cut; `none` is a flat zoom in a rounded card (byte-identical to `v2.1.0` on Windows and Linux; on Windows every checked frame is byte-identical to the PR build). `exportVideo` at 1080p30: 10.00 s, 1920×1080 H.264 with an AAC track, decodes without error, and its frame at 5 s shows the tilted card with rounded corners. Before the cut, on a dev build of the PR: the four device frames, dual-frame and vertical-stack, under `orbit`, `left` and `none`, rendered byte-identical to `v2.1.0` (49 of 49 frames). **Not covered:** recording, playback and the editor preview, which the patch does not touch; the rest of the checklist (the `v2.1.0-rc.6` rows). | | 2026-10-19 | `v2.1.2-rc.1`, `release/v2.1.2` @ `c4ef8fd` (= `v2.1.1` + #1344), assets of the GitHub pre-release (build run 39224311283). Windows: `Openscreen.Setup.2.1.2-rc.1-x64.exe` unpacked with 7-Zip and run in place. macOS: `Openscreen-Mac-arm64-2.1.2-rc.1.zip`, run in place. Linux: the AppImage, run directly with `--no-sandbox` | Windows 11 Home 26200; macOS 27.0.1, Mac mini M1; Ubuntu 24.04 GNOME Wayland (Xwayland `:0`) | **Partial: fix slice only, no defect** | `openscreen mcp` over stdio, scripted, with a fresh `--user-data-dir` per run. On all three platforms it answered `initialize`, and `usage-ping.json` held today's day, which is saved only once PostHog accepted the event. With `DO_NOT_TRACK=1` no state file was written. stdout held only JSON-RPC on macOS and Linux. On Windows it starts with one `\r\n` that Electron 41 itself writes during browser-process startup, before `main.js` loads: a bare `app.exit(0)` script prints it too, `ELECTRON_RUN_AS_NODE` does not. It predates this change, and the MCP SDK clients skip it with one parse error. Not covered: everything else in this file, including recording, editing, export, the GUI ping and the tray toggle. These runs sent eight real `surface: mcp` events to PostHog on 19/10. | +| 2026-10-21 | PR #1177, rebased on `main` @ `1e917109`, token presentation follow-up (dev build, Electron 41.2.1) | macOS 15.8, Apple Silicon | **Partial: Agent usage presentation** | Real Electron driven through OS computer-use, isolated `/tmp` profile and a local scripted OpenAI-compatible provider (no paid provider). A two-call turn showed muted `≥10.3k tokens` beside the activity indicator, then `12.3k tokens · 15s` beneath the reply. Input/output/cache details remained in the tooltip. During a second turn, creating Conversation 2 hid the running indicator/count there; returning to Conversation 1 restored the running count and then exactly one completed reply. Fully quit/relaunched Electron and reopened the editor: both completed summaries and their durations restored in Conversation 1, with old rewind controls absent. Sidecar inspection confirmed Conversation 2 remained empty. Skipped: capture/preview/export, native binaries, paid-provider billing, Windows/Linux and the rest of this checklist; chat-only change, not release-promotion QA. | +| 2026-10-22 | PR #1177, maintainer presentation follow-up on `54ee4e15` (dev build, Electron 41.2.1) | macOS 15.8, Apple Silicon | **Partial: Agent usage presentation** | Real Electron driven through OS computer-use with an isolated `/tmp` profile and a local scripted Anthropic-compatible provider (synthetic counts, no paid provider). A two-call turn showed muted `≥10.3k tokens` beside the spinner, one expandable Thinking label while reasoning streamed, then `12.3k tokens · 45s` beneath the reply. Captured [running](screenshots/pr-1177/running.png) and [completed](screenshots/pr-1177/completed.png) screenshots. Component tests also cover partial totals with details confined to the tooltip, unavailable usage showing duration only, and unavailable usage without duration rendering nothing. Skipped: capture/preview/export, native binaries, paid-provider billing, restart/session switching (covered in the preceding pass), Windows/Linux and the rest of this checklist; chat presentation only. | diff --git a/technical-documentation/testing/screenshots/pr-1177/completed.png b/technical-documentation/testing/screenshots/pr-1177/completed.png new file mode 100644 index 000000000..a34ba007e Binary files /dev/null and b/technical-documentation/testing/screenshots/pr-1177/completed.png differ diff --git a/technical-documentation/testing/screenshots/pr-1177/running.png b/technical-documentation/testing/screenshots/pr-1177/running.png new file mode 100644 index 000000000..e59e0bd15 Binary files /dev/null and b/technical-documentation/testing/screenshots/pr-1177/running.png differ