diff --git a/apps/desktop/e2e/transcript-scroll.spec.ts b/apps/desktop/e2e/transcript-scroll.spec.ts index d58cc1a61a..2a7fa86772 100644 --- a/apps/desktop/e2e/transcript-scroll.spec.ts +++ b/apps/desktop/e2e/transcript-scroll.spec.ts @@ -429,8 +429,12 @@ test('a gesture a nested scroller consumed does not release the tail', async ({ await page.setViewportSize({ width: 900, height: 700 }); await sendPrompt(page, LONG_PROMPT); await expect(answeredTurns(page)).toHaveCount(1, { timeout: 30_000 }); - const settled = await scrollMetrics(page); - expect(settled.distance, JSON.stringify(settled)).toBeLessThanOrEqual(4); + // The turn's arrival and the tail-follow write are two steps, so one sample + // races the follow on a loaded runner. Poll until the reader has provably + // been carried back to the tail. + await expect.poll(async () => (await scrollMetrics(page)).distance, { + message: 'the transcript follows the landed answer to the tail', + }).toBeLessThanOrEqual(4); // A real scroller inside the transcript, standing in for a tool-output box // (`.maka-tool-output-body`, `max-height: 256px; overflow-y: auto`) or a pty @@ -500,9 +504,23 @@ test('a nested scroller near the history boundary does not request an earlier ra }) => { await page.setViewportSize({ width: 900, height: 1500 }); await waitForPaintedFrames(page, 6); - const metrics = await scrollMetrics(page); - expect(metrics.scrollTop).toBeLessThanOrEqual(Math.max(640, metrics.clientHeight * 2)); - expect(metrics.distance).toBeLessThanOrEqual(4); + // The fixture is ready when the transcript exists, before its initial tail + // positioning necessarily completes. Poll one geometry sample so the pin has + // provably settled inside the load band and at the tail before the nested + // scroller exercises it. + await expect.poll(async () => { + const metrics = await scrollMetrics(page); + return { + insideLoadBand: metrics.scrollTop <= Math.max(640, metrics.clientHeight * 2), + settledAtTail: metrics.distance <= 4, + metrics, + }; + }, { + message: 'the initial transcript tail positioning settles', + }).toMatchObject({ + insideLoadBand: true, + settledAtTail: true, + }); const nestedBefore = await page.evaluate((selector) => { const root = document.querySelector(selector); @@ -706,12 +724,22 @@ test('following the tail does not ask for the history above it', async ({ await page.setViewportSize({ width: 900, height: 1500 }); await waitForPaintedFrames(page, 6); - const settled = await scrollMetrics(page); - expect(settled.distance, JSON.stringify(settled)).toBeLessThanOrEqual(4); - expect( - settled.scrollTop, - `the tail must be inside the load band for this test to mean anything: ${JSON.stringify(settled)}`, - ).toBeLessThanOrEqual(Math.max(640, settled.clientHeight * 2)); + // Same as above: the fixture being ready does not mean the initial tail + // positioning has completed. Poll until the pin has provably settled at the + // tail and inside the load band this test's history claim rests on. + await expect.poll(async () => { + const settled = await scrollMetrics(page); + return { + insideLoadBand: settled.scrollTop <= Math.max(640, settled.clientHeight * 2), + settledAtTail: settled.distance <= 4, + settled, + }; + }, { + message: 'the initial transcript tail positioning settles', + }).toMatchObject({ + insideLoadBand: true, + settledAtTail: true, + }); // Nothing arrived that the reader did not ask for. await waitForPaintedFrames(page, 12); diff --git a/apps/desktop/src/main/__tests__/runtime-host-usage-ipc-main.test.ts b/apps/desktop/src/main/__tests__/runtime-host-usage-ipc-main.test.ts index 3b7b3df61f..3dcd30173d 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-usage-ipc-main.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-usage-ipc-main.test.ts @@ -56,6 +56,7 @@ test("settings usage stats use the canonical model-call total and load every act cacheHitRequests: 10, cacheCreateRequests: 5, errorRequests: 2, + totalDurationMs: 0, }, provenance: provenance(), } satisfies UsageQueryResult; @@ -176,6 +177,7 @@ test("settings usage stats reject a non-advancing activity page", async () => { cacheHitRequests: 0, cacheCreateRequests: 0, errorRequests: 0, + totalDurationMs: 0, }, provenance: provenance(), } satisfies UsageQueryResult; @@ -243,6 +245,7 @@ test("settings usage stats degrade instead of erroring when logs disagree with t cacheHitRequests: 0, cacheCreateRequests: 0, errorRequests: 0, + totalDurationMs: 0, }, provenance: provenance(), } satisfies UsageQueryResult; @@ -316,6 +319,7 @@ test("settings usage stats group the provider breakdown by connection", async () cacheHitRequests: 0, cacheCreateRequests: 0, errorRequests: 0, + totalDurationMs: 0, }, provenance: provenance(), } satisfies UsageQueryResult; @@ -394,6 +398,7 @@ test("settings usage stats truncate the activity log at the cap instead of error cacheHitRequests: 0, cacheCreateRequests: 0, errorRequests: 0, + totalDurationMs: 0, }, provenance: provenance(), } satisfies UsageQueryResult; @@ -467,6 +472,7 @@ test("settings usage stats name each row from the Host-resolved session title", cacheHitRequests: 0, cacheCreateRequests: 0, errorRequests: 0, + totalDurationMs: 0, }, provenance: provenance(), } satisfies UsageQueryResult; diff --git a/apps/desktop/src/main/__tests__/session-inspector-panel-model.test.ts b/apps/desktop/src/main/__tests__/session-inspector-panel-model.test.ts index e301fd4ec7..1b39a27bc0 100644 --- a/apps/desktop/src/main/__tests__/session-inspector-panel-model.test.ts +++ b/apps/desktop/src/main/__tests__/session-inspector-panel-model.test.ts @@ -50,6 +50,7 @@ test('does not render legacy zero cost as a known free Session', () => { cacheHitRequests: 0, cacheCreateRequests: 0, errorRequests: 0, + totalDurationMs: 0, provenance: { coverage: { attempts: 0, @@ -85,6 +86,7 @@ test('reports incomplete provenance as unavailable regardless of recorded reques cacheHitRequests: 0, cacheCreateRequests: 0, errorRequests: 0, + totalDurationMs: 0, provenance: { coverage: { attempts: 0, @@ -121,6 +123,7 @@ test('does not estimate a cache-hit ratio from partial usage', () => { cacheHitRequests: 1, cacheCreateRequests: 0, errorRequests: 0, + totalDurationMs: 0, provenance: { coverage: { attempts: 1, diff --git a/apps/desktop/src/main/__tests__/session-inspector-usage-stats.test.ts b/apps/desktop/src/main/__tests__/session-inspector-usage-stats.test.ts new file mode 100644 index 0000000000..bfb0811487 --- /dev/null +++ b/apps/desktop/src/main/__tests__/session-inspector-usage-stats.test.ts @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + deriveInspectorOverviewModel, + RING_ACTIVE_MIN_SWEEP, + RING_MIN_SWEEP, + usageRingArcs, +} from '../../renderer/features/workbar/testing.js'; +import type { UsageSummaryV2 } from '@maka/core/usage-stats/types'; + +function usageSummary(overrides: Partial = {}): UsageSummaryV2 { + return { + range: { from: 0, to: 1 }, + totalRequests: 3, + totalCostUsd: 0.02, + totalTokens: { + input: 4_000_000, + output: 60_300, + cacheMiss: 100_000, + cacheRead: 3_900_000, + cacheWrite: 0, + reasoning: 12_000, + total: 4_060_300, + }, + cacheHitRequests: 2, + cacheCreateRequests: 0, + errorRequests: 0, + totalDurationMs: 0, + ...overrides, + }; +} + +test('splits the session metered tokens the way a bill reads', () => { + const { tokenUsage } = deriveInspectorOverviewModel( + undefined, + usageSummary({ totalDurationMs: 1_885_000 }), + ); + + assert.ok(tokenUsage); + assert.deepEqual( + tokenUsage.segments.map((segment) => [segment.kind, segment.tokens]), + [ + ['cacheRead', 3_900_000], + ['cacheMiss', 100_000], + ['output', 60_300], + ], + ); + // The readout is the sum of the drawn rows, so the legend and its total + // cannot disagree even when the ledger's own parts drifted. + assert.equal( + tokenUsage.total, + tokenUsage.segments.reduce((carry, segment) => carry + segment.tokens, 0), + ); +}); + +test('derives uncached input as the prompt residual when a provider reports only its cache', () => { + // A cache-reading-only provider leaves the ledger's cacheMiss at zero; the + // row must still carry what the session paid as uncached input. + const { tokenUsage } = deriveInspectorOverviewModel( + undefined, + usageSummary({ + totalTokens: { + input: 260_500, + output: 60_300, + cacheMiss: 0, + cacheRead: 200_000, + cacheWrite: 0, + reasoning: 0, + total: 320_800, + }, + }), + ); + + assert.deepEqual( + tokenUsage?.segments.map((segment) => [segment.kind, segment.tokens]), + [ + ['cacheRead', 200_000], + ['cacheMiss', 60_500], + ['output', 60_300], + ], + ); +}); + +test('keeps the ledger cacheMiss as a floor for records that reported no prompt total', () => { + const { tokenUsage } = deriveInspectorOverviewModel( + undefined, + usageSummary({ + totalTokens: { + input: 0, + output: 100, + cacheMiss: 500, + cacheRead: 0, + cacheWrite: 0, + reasoning: 0, + total: 100, + }, + }), + ); + + assert.deepEqual( + tokenUsage?.segments.map((segment) => [segment.kind, segment.tokens]), + [ + ['cacheMiss', 500], + ['output', 100], + ], + ); +}); + +test('a session with nothing metered has no token split to show', () => { + const { tokenUsage } = deriveInspectorOverviewModel( + undefined, + usageSummary({ + totalTokens: { + input: 0, + output: 0, + cacheMiss: 0, + cacheRead: 0, + cacheWrite: 0, + reasoning: 0, + total: 0, + }, + }), + ); + assert.equal(tokenUsage, undefined); + assert.equal(deriveInspectorOverviewModel(undefined, undefined).tokenUsage, undefined); +}); + +test('shows the token split even when usage coverage is partial, since the rows are what ran', () => { + // The cache-hit RATE goes unavailable under partial usage — a rate over a + // part is a lie. The split stays: its rows state what was recorded, which + // only undercounts, never fabricates. + const { tokenUsage, cacheHitRate } = deriveInspectorOverviewModel(undefined, { + ...usageSummary(), + provenance: { + coverage: { + attempts: 3, + pricedAttempts: 3, + unpricedAttempts: 0, + usageReportedAttempts: 2, + usagePartialAttempts: 1, + usageMissingAttempts: 0, + }, + legacyRecords: 0, + unreadableRecords: 0, + pendingRepairs: 0, + }, + }); + assert.ok(tokenUsage); + assert.equal(cacheHitRate, undefined); +}); + +test('splits recorded time between model calls and tool executions', () => { + const { durationUsage } = deriveInspectorOverviewModel( + undefined, + usageSummary({ totalDurationMs: 1_873_000, toolUsage: { requests: 78, durationMs: 78_000 } }), + ); + + assert.deepEqual( + durationUsage?.segments.map((segment) => [segment.kind, segment.count, segment.durationMs]), + [ + ['model', 3, 1_873_000], + ['tool', 78, 78_000], + ], + ); + assert.equal(durationUsage?.totalDurationMs, 1_873_000 + 78_000); +}); + +test('keeps a cache-read share the provider reported without a prompt total', () => { + // Core leaves the ledger's cacheRead unclamped when no prompt total was + // reported, so cacheRead > input is a normal aggregate there — clamping it + // to input would erase exactly the sessions the split exists to describe. + const { tokenUsage } = deriveInspectorOverviewModel( + undefined, + usageSummary({ + totalTokens: { + input: 0, + output: 100, + cacheMiss: 60_000, + cacheRead: 200_000, + cacheWrite: 0, + reasoning: 0, + total: 260_100, + }, + }), + ); + + assert.deepEqual( + tokenUsage?.segments.map((segment) => [segment.kind, segment.tokens]), + [ + ['cacheRead', 200_000], + ['cacheMiss', 60_000], + ['output', 100], + ], + ); + assert.equal(tokenUsage?.total, 260_100); +}); + +test('a host that measured zero model time keeps its row and its call count', () => { + // Presence follows the reported field, not a zero-derived default: zero is + // a measurement, and the row carries the count the model totals show too. + const { durationUsage } = deriveInspectorOverviewModel( + undefined, + usageSummary({ totalDurationMs: 0 }), + ); + assert.deepEqual( + durationUsage?.segments.map((segment) => [segment.kind, segment.count, segment.durationMs]), + [['model', 3, 0]], + ); +}); + +test('a row with neither a clock nor a count is dropped', () => { + const { durationUsage } = deriveInspectorOverviewModel( + undefined, + usageSummary({ totalRequests: 0, totalDurationMs: 0 }), + ); + assert.equal(durationUsage, undefined); +}); + +test('a tool row without a recorded duration still reports its count', () => { + const { durationUsage } = deriveInspectorOverviewModel( + undefined, + usageSummary({ totalRequests: 0, totalDurationMs: 0, toolUsage: { requests: 4, durationMs: 0 } }), + ); + + assert.deepEqual( + durationUsage?.segments.map((segment) => [segment.kind, segment.count, segment.durationMs]), + [['tool', 4, 0]], + ); + assert.equal(durationUsage?.totalDurationMs, 0); +}); + +test('model time without tool usage reads as a single-segment split', () => { + const { durationUsage } = deriveInspectorOverviewModel( + undefined, + usageSummary({ totalRequests: 5, totalDurationMs: 2_500 }), + ); + + assert.deepEqual( + durationUsage?.segments.map((segment) => [segment.kind, segment.count, segment.durationMs]), + [['model', 5, 2_500]], + ); +}); + +test('ring arcs keep reading order and clamp the last segment to the full turn', () => { + const arcs = usageRingArcs( + [ + { kind: 'cacheRead' as const, amount: 1 }, + { kind: 'cacheMiss' as const, amount: 1 }, + { kind: 'output' as const, amount: 1 }, + ], + 3, + ); + assert.deepEqual( + arcs.map((arc) => arc.kind), + ['cacheRead', 'cacheMiss', 'output'], + ); + assert.equal(arcs[0]?.start, 0); + // The last arc is clamped to the full turn rather than accumulated, so a + // rounded share can never leave an unexplained sliver at the seam. + assert.equal(arcs.at(-1)?.end, 1); + for (const arc of arcs) { + assert.ok(arc.end > arc.start); + assert.match(arc.d, /^M [\d. ]+ A/); + assert.match(arc.d, / Z$/); + } +}); + +test('a segment owning the whole ring walks two half-turns instead of one arc', () => { + const arcs = usageRingArcs([{ kind: 'model' as const, amount: 2_500 }], 2_500); + assert.equal(arcs.length, 1); + assert.equal(arcs[0]?.start, 0); + assert.equal(arcs[0]?.end, 1); + // One arc cannot sweep 360°; two half arcs render the full donut. + assert.equal(arcs[0]?.d.match(/ A /g)?.length, 4); +}); + +test('a ring with nothing measured draws no arcs and leaves the muted track', () => { + assert.equal(usageRingArcs([{ kind: 'tool' as const, amount: 0 }], 0).length, 0); + assert.equal(usageRingArcs([], 100).length, 0); +}); + +test('a nonzero sliver keeps a visible sweep even when its share rounds to zero', () => { + const arcs = usageRingArcs( + [ + { kind: 'model' as const, amount: 77_000 }, + { kind: 'tool' as const, amount: 35 }, + ], + 77_035, + ); + const tool = arcs[1]; + // The end-of-turn clamp re-derives the last sweep from the running cursor, + // so compare with the usual floating-point courtesy. + assert.ok(tool.end - tool.start >= RING_MIN_SWEEP - 1e-9); + assert.equal(tool.end, 1); +}); + +test('hovering a tiny segment expands it on the ring so the highlight lands somewhere', () => { + const arcs = usageRingArcs( + [ + { kind: 'model' as const, amount: 77_000 }, + { kind: 'tool' as const, amount: 35 }, + ], + 77_035, + 'tool', + ); + const tool = arcs[1]; + // The end-of-turn clamp re-derives the last sweep from the running cursor, + // so compare with the usual floating-point courtesy. + assert.ok(tool.end - tool.start >= RING_ACTIVE_MIN_SWEEP - 1e-9); + // The cost of the focus floor comes out of the dominant share, and the + // seam still closes on the full turn. + assert.ok(arcs[0].end <= 1 - RING_ACTIVE_MIN_SWEEP + 1e-9); + assert.equal(tool.end, 1); +}); + +test('hovering the dominant segment leaves the layout untouched', () => { + const arcs = usageRingArcs( + [ + { kind: 'model' as const, amount: 77_000 }, + { kind: 'tool' as const, amount: 35 }, + ], + 77_035, + 'model', + ); + assert.ok(arcs[0].end >= 1 - RING_MIN_SWEEP); + assert.equal(arcs[1].end, 1); +}); diff --git a/apps/desktop/src/main/__tests__/use-session-trace.test.ts b/apps/desktop/src/main/__tests__/use-session-trace.test.ts index 06d0b5eed1..75c1437194 100644 --- a/apps/desktop/src/main/__tests__/use-session-trace.test.ts +++ b/apps/desktop/src/main/__tests__/use-session-trace.test.ts @@ -79,6 +79,7 @@ function usageSummary( cacheHitRequests: 0, cacheCreateRequests: 0, errorRequests: 0, + totalDurationMs: 0, provenance: { coverage: { attempts: totalRequests, diff --git a/apps/desktop/src/renderer/features/workbar/testing.ts b/apps/desktop/src/renderer/features/workbar/testing.ts index a6c30b119e..6604a4f050 100644 --- a/apps/desktop/src/renderer/features/workbar/testing.ts +++ b/apps/desktop/src/renderer/features/workbar/testing.ts @@ -32,7 +32,13 @@ export * from './model/workbar-tool-definitions.js'; export * from './tools/artifacts/artifact-list-keyboard.js'; export * from './tools/artifacts/artifact-visibility.js'; export * from './tools/inspector/session-inspector-panel-model.js'; -export { compactNumberFormatter, InspectorCompositionSection } from './tools/inspector/session-inspector-panel.js'; +export { + compactNumberFormatter, + InspectorCompositionSection, + RING_ACTIVE_MIN_SWEEP, + RING_MIN_SWEEP, + usageRingArcs, +} from './tools/inspector/session-inspector-panel.js'; export * from './tools/inspector/session-inspector-overview-model.js'; export * from './tools/side-chat/quote-companion-panel-state.js'; export * from './tools/side-chat/quote-companion-core.js'; diff --git a/apps/desktop/src/renderer/features/workbar/tools/inspector/session-inspector-overview-model.ts b/apps/desktop/src/renderer/features/workbar/tools/inspector/session-inspector-overview-model.ts index 36e835d81f..b4af6d30cd 100644 --- a/apps/desktop/src/renderer/features/workbar/tools/inspector/session-inspector-overview-model.ts +++ b/apps/desktop/src/renderer/features/workbar/tools/inspector/session-inspector-overview-model.ts @@ -140,6 +140,55 @@ export interface InspectorOverviewModel { * the run ledger; three statements of the same tokens is two too many. */ cacheHitRate?: number; + /** + * The session's metered tokens split the way a bill reads: served from the + * provider's cache, paid as uncached input, paid as output. Absent when + * nothing was metered — a breakdown of zero tokens is not zeros, it is + * nothing to break down. + */ + tokenUsage?: InspectorTokenUsage; + /** + * Where the session's recorded time went, model calls against tool + * executions. Absent when the query could not answer either ledger — a + * connection-scoped query omits the tool side, and a session with no + * recorded time has no split to draw. + */ + durationUsage?: InspectorDurationUsage; +} + +export type InspectorTokenUsageKind = 'cacheRead' | 'cacheMiss' | 'output'; + +export interface InspectorTokenUsageSegment { + kind: InspectorTokenUsageKind; + tokens: number; +} + +export interface InspectorTokenUsage { + /** cacheRead + uncached input + output — the metered, billable tokens. */ + total: number; + /** + * In reading order with empty rows dropped, so the bands and their legend + * cannot disagree about what is drawn. + */ + segments: readonly InspectorTokenUsageSegment[]; +} + +export type InspectorDurationUsageKind = 'model' | 'tool'; + +export interface InspectorDurationUsageSegment { + kind: InspectorDurationUsageKind; + count: number; + durationMs: number; +} + +export interface InspectorDurationUsage { + /** + * model + tool recorded time. A sum of per-call durations, not wall-clock: + * parallel settlements and nested calls overlap, so this can exceed the + * session's elapsed time. + */ + totalDurationMs: number; + segments: readonly InspectorDurationUsageSegment[]; } export function estimatedSessionCost( @@ -182,10 +231,14 @@ export function deriveInspectorOverviewModel( const composition = compositionState(diagnostics); const context = contextBudget(diagnostics); const cacheHitRate = usageCacheHitRate(usage); + const tokenUsage = usageTokenSplit(usage); + const durationUsage = usageDurationSplit(usage); return { ...(context ? { context } : {}), ...(composition ? { composition } : {}), ...(cacheHitRate !== undefined ? { cacheHitRate } : {}), + ...(tokenUsage ? { tokenUsage } : {}), + ...(durationUsage ? { durationUsage } : {}), }; } @@ -201,6 +254,75 @@ function usageCacheHitRate(usage: SessionUsageSummary | undefined): number | und return usage.totalTokens.cacheRead / usage.totalTokens.input; } +/** + * The bill-shaped split of the session's metered tokens. + * + * `cacheMiss` is computed as the residual `input − cacheRead` rather than read + * from the ledger: several providers report the cached share only, and a + * ledger zero there would understate the uncached input the session paid for. + * The ledger's own `cacheMiss` is kept as a floor for records that reported + * the miss but no prompt total. `cacheRead` is taken as reported — providers + * that itemize the cached share without a prompt total leave it larger than + * `input`, and clamping it here would erase exactly the sessions the split + * exists to describe. Output is the metered figure as billed — providers that + * itemize reasoning include it in it, which is what the panel's row label says. + */ +function usageTokenSplit(usage: SessionUsageSummary | undefined): InspectorTokenUsage | undefined { + if (!usage) return undefined; + const { input, output, cacheRead, cacheMiss } = usage.totalTokens; + if (input + output <= 0) return undefined; + const uncachedInput = Math.max(input - cacheRead, cacheMiss, 0); + const segments = [ + { kind: 'cacheRead' as const, tokens: cacheRead }, + { kind: 'cacheMiss' as const, tokens: uncachedInput }, + { kind: 'output' as const, tokens: output }, + ].filter((segment) => segment.tokens > 0); + // The readout is the sum of what is drawn, so the rows and their total + // cannot disagree even when a provider's report made the ledger's own + // `total` drift from its parts. + return { + total: segments.reduce((carry, segment) => carry + segment.tokens, 0), + segments, + }; +} + +/** + * Model-call time against tool-execution time, both session-wide from their + * own ledgers. Presence follows the reported fields, not a zero default: a + * host that measured zero model time keeps its row — the call count is real — + * and `toolUsage` is simply absent when the query could not be scoped. A row + * with neither a clock nor a count is dropped; a split nobody measured is not + * a split worth drawing. + */ +function usageDurationSplit( + usage: SessionUsageSummary | undefined, +): InspectorDurationUsage | undefined { + if (!usage) return undefined; + const segments = ( + [ + { + kind: 'model' as const, + count: usage.totalRequests, + durationMs: usage.totalDurationMs, + }, + ...(usage.toolUsage + ? [ + { + kind: 'tool' as const, + count: usage.toolUsage.requests, + durationMs: usage.toolUsage.durationMs, + }, + ] + : []), + ] satisfies InspectorDurationUsageSegment[] + ).filter((segment) => segment.durationMs > 0 || segment.count > 0); + if (segments.length === 0) return undefined; + return { + totalDurationMs: segments.reduce((carry, segment) => carry + segment.durationMs, 0), + segments, + }; +} + /** * The budget question a reader actually asks — "how full is the context right * now" — answered by the snapshot's own metered prompt against the window that diff --git a/apps/desktop/src/renderer/features/workbar/tools/inspector/session-inspector-panel.tsx b/apps/desktop/src/renderer/features/workbar/tools/inspector/session-inspector-panel.tsx index 86749b04dc..ac29081afb 100644 --- a/apps/desktop/src/renderer/features/workbar/tools/inspector/session-inspector-panel.tsx +++ b/apps/desktop/src/renderer/features/workbar/tools/inspector/session-inspector-panel.tsx @@ -17,7 +17,7 @@ * under the License. */ -import { type ReactNode, useMemo } from 'react'; +import { type ReactNode, useMemo, useState } from 'react'; import { Banner } from '@astryxdesign/core/Banner'; import { Button } from '@astryxdesign/core/Button'; import { EmptyState } from '@astryxdesign/core/EmptyState'; @@ -38,6 +38,8 @@ import { deriveInspectorOverviewModel, estimatedSessionCost, hasUnavailableSessionUsage, + type InspectorDurationUsageKind, + type InspectorTokenUsageKind, } from './session-inspector-overview-model.js'; import { deriveInspectorPanelModel, @@ -269,9 +271,67 @@ function InspectorOverview(props: { const formatNumber = numberFormatter(props.locale); const formatCompactNumber = compactNumberFormatter(props.locale); const context = overview.context; + // Local bindings so the JSX guards narrow into the map callbacks below. + const tokenUsage = overview.tokenUsage; + const durationUsage = overview.durationUsage; + // The token ring's center names whichever share dominates the bill. + const dominantToken = tokenUsage?.segments.reduce((left, right) => + right.tokens > left.tokens ? right : left, + ); return ( + {/* The two session-wide ledgers open the panel — they are what a reader + scanning "what did this session cost so far" wants first, ahead of + the cost figures they extend and the context bar, which is the one + block here that answers about NOW rather than about the session so + far. */} + {tokenUsage && dominantToken && ( + ({ + kind: segment.kind, + amount: segment.tokens, + }))} + total={tokenUsage.total} + centerValue={formatPercent(dominantToken.tokens / tokenUsage.total)} + centerLabel={copy.tokenUsage.segment[dominantToken.kind]} + rows={tokenUsage.segments.map((segment) => ({ + kind: segment.kind, + label: copy.tokenUsage.segment[segment.kind], + swatch: `token-${segment.kind}`, + value: `${formatCompactNumber(segment.tokens)} · ${formatPercent( + segment.tokens / tokenUsage.total, + )}`, + }))} + /> + )} + + {durationUsage && ( + ({ + kind: segment.kind, + amount: segment.durationMs, + }))} + total={durationUsage.totalDurationMs} + centerValue={formatDuration(durationUsage.totalDurationMs)} + centerLabel={copy.durationUsage.center} + rows={durationUsage.segments.map((segment) => ({ + kind: segment.kind, + label: copy.durationUsage.segment[segment.kind](segment.count), + swatch: `duration-${segment.kind}`, + value: `${formatDuration(segment.durationMs)} · ${formatPercent( + durationUsage.totalDurationMs > 0 + ? segment.durationMs / durationUsage.totalDurationMs + : 0, + )}`, + }))} + /> + )} + {props.showTotals && ( (props: { + contract: string; + title: string; + arcs: readonly { kind: K; amount: number }[]; + total: number; + centerValue: ReactNode; + centerLabel: ReactNode; + rows: readonly { kind: K; label: ReactNode; swatch: string; value: ReactNode }[]; +}) { + const [active, setActive] = useState(null); + return ( + +
+ + {props.title} + +
+ +
+
+ + + {props.centerValue} + {props.centerLabel} + +
+
+ {props.rows.map((row) => ( +
+
+ + ); +} + +/** + * The donut itself: one muted track underneath, one arc per segment over it. + * + * Hover state lives in the section (arcs and legend rows both need it), so + * this is a controlled leaf — `active` names the held segment, and every + * other arc drops back rather than competing for the eye. The svg is + * `aria-hidden`: the legend beside it is the accessible copy of the ring. + */ +const RING_OUTER = 50; +const RING_INNER = 32; +const RING_TRACK_RADIUS = (RING_OUTER + RING_INNER) / 2; + +function UsageRing< + K extends InspectorTokenUsageKind | InspectorDurationUsageKind, +>(props: { + arcs: readonly { kind: K; amount: number }[]; + total: number; + active: K | null; + onActive: (kind: K | null) => void; +}) { + const arcs = usageRingArcs(props.arcs, props.total, props.active); + return ( + + ); +} + +/** + * The smallest slice a segment draws: a hairline. A share that rounds to zero + * — 35ms of tool time against 77s of model calls — would otherwise sweep + * less than a degree and vanish, and a legend row pointing at nothing on the + * ring reads as a bug. The hovered segment draws a thin line instead — a step + * above the hairline, still nowhere near a real share — so the highlight has + * somewhere to land; that is focus feedback, not a re-measurement, and the + * legend beside it keeps the true figures. + */ +export const RING_MIN_SWEEP = 0.5 / 360; +export const RING_ACTIVE_MIN_SWEEP = 3 / 360; + +/** + * One arc per segment, in reading order. Any share under its floor is widened + * to it, taking the difference from the shares still above theirs in + * proportion; the last segment's end is clamped to the full turn, so neither + * the floors nor rounding can leave an unexplained sliver at the seam. A ring + * with nothing to measure draws no arcs at all and leaves the muted track + * visible. + */ +export function usageRingArcs( + items: readonly { kind: K; amount: number }[], + total: number, + active?: K | null, +): readonly { kind: K; start: number; end: number; d: string }[] { + const drawn = items.filter((item) => item.amount > 0); + if (drawn.length === 0 || total <= 0) return []; + const values = drawn.map((item) => item.amount / total); + const mins = drawn.map((item) => + item.kind === active ? RING_ACTIVE_MIN_SWEEP : RING_MIN_SWEEP, + ); + for (let round = 0; round < 2; round += 1) { + let deficit = 0; + let donorBudget = 0; + for (let i = 0; i < values.length; i += 1) { + if (values[i] < mins[i]) deficit += mins[i] - values[i]; + else donorBudget += values[i] - mins[i]; + } + if (deficit === 0 || donorBudget <= 0) break; + for (let i = 0; i < values.length; i += 1) { + if (values[i] < mins[i]) values[i] = mins[i]; + else values[i] = mins[i] + (values[i] - mins[i]) * (1 - deficit / donorBudget); + } + } + let cursor = 0; + return drawn.map((item, index) => { + const start = cursor; + const end = index === drawn.length - 1 ? 1 : Math.min(1, cursor + values[index]); + cursor = end; + return { kind: item.kind, start, end, d: ringSectorPath(start, end) }; + }); +} + +/** A point on the ring, as SVG coordinates. Fraction 0 sits at 12 o'clock. */ +function ringPoint(radius: number, fraction: number): string { const angle = fraction * 2 * Math.PI; + const x = 50 + radius * Math.sin(angle); + const y = 50 - radius * Math.cos(angle); + return `${x.toFixed(3)} ${y.toFixed(3)}`; +} + +/** + * A donut sector between two fractions of one turn. A single arc cannot + * sweep a full circle, so a segment owning everything walks two half-turns + * instead — same fill, hairline seam at 12 o'clock where the reading starts. + */ +function ringSectorPath(from: number, to: number): string { + const turns: readonly (readonly [number, number])[] = + to - from >= 0.999_99 ? [[from, from + 0.5], [from + 0.5, to]] : [[from, to]]; + let d = ''; + for (const [start, end] of turns) { + const largeArc = end - start > 0.5 ? 1 : 0; + d += `M ${ringPoint(RING_OUTER, start)} A ${RING_OUTER} ${RING_OUTER} 0 ${largeArc} 1 ${ringPoint(RING_OUTER, end)} `; + d += `L ${ringPoint(RING_INNER, end)} A ${RING_INNER} ${RING_INNER} 0 ${largeArc} 0 ${ringPoint(RING_INNER, start)} Z `; + } + return d.trim(); +} + /** * One legend row: band, figure. * @@ -334,10 +578,27 @@ function InspectorOverviewStat(props: { label: string; value: ReactNode }) { * number columns per row was what made this read as a spreadsheet; what is * left is name-left / number-right, the same skeleton as a step row, so the * whole panel scans on one rhythm. + * + * `active`/`dimmed`/`onHover` are the ring-linking hooks: hovering a row + * lights its arc and vice versa. They are opt-in, so the context and + * composition legends stay non-interactive. */ -function FactRow(props: { label: ReactNode; value: ReactNode; swatch?: ReactNode }) { +function FactRow(props: { + label: ReactNode; + value: ReactNode; + swatch?: ReactNode; + active?: boolean; + dimmed?: boolean; + onHover?: (hovered: boolean) => void; +}) { return ( -
+
props.onHover?.(true) : undefined} + onMouseLeave={props.onHover ? () => props.onHover?.(false) : undefined} + >
{props.swatch} {props.label} diff --git a/apps/desktop/src/renderer/locales/conversation-copy.ts b/apps/desktop/src/renderer/locales/conversation-copy.ts index 05fba0ff9c..797dda1f15 100644 --- a/apps/desktop/src/renderer/locales/conversation-copy.ts +++ b/apps/desktop/src/renderer/locales/conversation-copy.ts @@ -217,6 +217,28 @@ export interface DesktopConversationCopy { totals: { cost: string; }; + /** + * The session-wide metered-token split, read like a bill: what the + * provider's cache served, what was paid as uncached input, what was paid + * as output. Names the bands of the token track, in the track's order. + */ + tokenUsage: { + title: string; + segment: { cacheRead: string; cacheMiss: string; output: string }; + }; + /** + * Where the session's recorded time went. Names the bands of the duration + * track; each row also states how many times its kind ran. + */ + durationUsage: { + title: string; + /** Label under the ring's total figure. */ + center: string; + segment: { + model: (count: number) => string; + tool: (count: number) => string; + }; + }; /** * The coverage notice, composed with its own breakdown: the separators * belong to the language, not to the layout, so a Chinese sentence gets @@ -579,6 +601,22 @@ const COPY = { totals: { cost: '估算成本', }, + tokenUsage: { + title: 'Token 统计', + segment: { + cacheRead: '缓存输入', + cacheMiss: '未命中输入', + output: '输出（含思考）', + }, + }, + durationUsage: { + title: '耗时统计', + center: '记录时长', + segment: { + model: (count) => `LLM 调用 × ${count}`, + tool: (count) => `工具执行 × ${count}`, + }, + }, coveragePartial: (parts) => `部分调用未能完整显示，下面的数字只少不多${zhDetail(parts)}`, coverageAbsent: (parts) => `这个后端不记录每次调用的明细${zhDetail(parts)}`, unreadable: (count) => `${count} 条记录读不出来`, @@ -819,6 +857,22 @@ const COPY = { totals: { cost: 'Estimated cost', }, + tokenUsage: { + title: 'Token usage', + segment: { + cacheRead: 'Cached input', + cacheMiss: 'Uncached input', + output: 'Output (incl. reasoning)', + }, + }, + durationUsage: { + title: 'Time breakdown', + center: 'Recorded Time', + segment: { + model: (count) => `LLM Calls × ${count}`, + tool: (count) => `Tool Runs × ${count}`, + }, + }, coveragePartial: (parts) => `Some calls could not be shown completely, so the numbers below only undercount${enDetail(parts)}`, coverageAbsent: (parts) => `This backend does not record per-call detail${enDetail(parts)}`, diff --git a/apps/desktop/src/renderer/styles/workbar/inspector.css b/apps/desktop/src/renderer/styles/workbar/inspector.css index 65ccd968f6..782d42ab0d 100644 --- a/apps/desktop/src/renderer/styles/workbar/inspector.css +++ b/apps/desktop/src/renderer/styles/workbar/inspector.css @@ -352,6 +352,170 @@ background: light-dark(oklch(0.66 0.15 75), oklch(0.78 0.13 75)); } +/* The session-wide token and time rings. The palette is defined once on the + panel so the legend swatches and each ring's inline conic-gradient read the + same five values, and a hue keeps one meaning inside the panel: green is + cache, violet is uncached input, blue is output — the duration ring reuses + blue/green for model/tool, matching the token ring's output/cache pairing + rather than inventing two more hues. */ +.maka-inspector-panel { + --_usage-cache-read: light-dark(oklch(0.61 0.14 165), oklch(0.75 0.12 165)); + --_usage-cache-miss: light-dark(oklch(0.62 0.16 315), oklch(0.74 0.13 315)); + --_usage-output: light-dark(oklch(0.62 0.18 255), oklch(0.72 0.15 255)); + --_usage-model: light-dark(oklch(0.62 0.18 255), oklch(0.72 0.15 255)); + --_usage-tool: light-dark(oklch(0.61 0.14 165), oklch(0.75 0.12 165)); +} + +/* Ring left, legend right. The ring is an SVG — one muted track under one + path per segment — because hover linking needs each segment to be its own + element; a conic-gradient disc is one box and cannot be pointed at. Fill + colours come from the panel palette above, so no hue is duplicated here. */ +.maka-inspector-usage-ring-row { + display: flex; + align-items: center; + gap: var(--space-4); +} + +.maka-inspector-usage-ring { + position: relative; + width: 7rem; + height: 7rem; + flex: 0 0 auto; +} + +.maka-inspector-usage-ring-svg { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + display: block; +} + +.maka-inspector-usage-ring-track { + fill: none; + stroke: var(--muted); + /* outer radius minus inner: the band the arcs paint into. */ + stroke-width: 18; +} + +.maka-inspector-usage-ring-path { + transition: opacity 120ms ease; +} + +.maka-inspector-usage-ring-path[data-dimmed='true'] { + opacity: 0.3; +} + +.maka-inspector-usage-ring-center { + /* Confined well inside the hole the mask cuts (64% of the ring), so the + label wraps as a clear of the band instead of touching it. */ + position: absolute; + inset: 24%; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: var(--space-1); + text-align: center; + text-wrap: balance; + /* The figure sits in the donut's hole; it must not swallow the hovers + meant for the arcs around it. */ + pointer-events: none; +} + +.maka-inspector-usage-ring-value { + font: var(--maka-text-body); + color: var(--foreground); + font-variant-numeric: tabular-nums; +} + +.maka-inspector-usage-ring-label { + font: var(--maka-text-supporting); + color: var(--muted-foreground); +} + +.maka-inspector-usage-legend { + flex: 1 1 auto; + min-width: 0; +} + +/* The two-way linking, legend side: the row the pointer (or an arc) holds + gains weight, everything else drops back. Colour ranks carry it — the same + three inks the panel already uses, no new tiers. */ +.maka-inspector-usage-legend .maka-inspector-usage-swatch { + transition: opacity 120ms ease; +} + +.maka-inspector-usage-legend .maka-inspector-grid-row[data-active] dt, +.maka-inspector-usage-legend .maka-inspector-grid-row[data-active] dd { + color: var(--foreground); + font-weight: 600; +} + +.maka-inspector-usage-legend .maka-inspector-grid-row[data-dimmed='true'] dt, +.maka-inspector-usage-legend .maka-inspector-grid-row[data-dimmed='true'] dd, +.maka-inspector-usage-legend .maka-inspector-grid-row[data-dimmed='true'] .maka-inspector-usage-swatch { + color: var(--muted-foreground); + opacity: 0.4; +} + +/* Below this width the legend rows would starve beside the ring, so the ring + moves above them instead. */ +@container maka-inspector (max-width: 26rem) { + .maka-inspector-usage-ring-row { + flex-direction: column; + align-items: flex-start; + } +} + +.maka-inspector-usage-swatch { + width: var(--space-1-5); + height: var(--space-1-5); + border-radius: var(--radius-pill); + flex: 0 0 auto; +} + +.maka-inspector-usage-swatch[data-usage='token-cacheRead'] { + background: var(--_usage-cache-read); +} + +.maka-inspector-usage-swatch[data-usage='token-cacheMiss'] { + background: var(--_usage-cache-miss); +} + +.maka-inspector-usage-swatch[data-usage='token-output'] { + background: var(--_usage-output); +} + +.maka-inspector-usage-swatch[data-usage='duration-model'] { + background: var(--_usage-model); +} + +.maka-inspector-usage-swatch[data-usage='duration-tool'] { + background: var(--_usage-tool); +} + +/* Arcs key on the bare kind — kinds are unique across both rings. */ +.maka-inspector-usage-ring-path[data-usage='cacheRead'] { + fill: var(--_usage-cache-read); +} + +.maka-inspector-usage-ring-path[data-usage='cacheMiss'] { + fill: var(--_usage-cache-miss); +} + +.maka-inspector-usage-ring-path[data-usage='output'] { + fill: var(--_usage-output); +} + +.maka-inspector-usage-ring-path[data-usage='model'] { + fill: var(--_usage-model); +} + +.maka-inspector-usage-ring-path[data-usage='tool'] { + fill: var(--_usage-tool); +} + /* The context window drawn as bands. Colour is the join between the track and its legend, so both read it from the same `data-segment` rules below and a band can never mean one thing in the bar and another in the list. */ diff --git a/apps/desktop/stories/session-workbar.stories.tsx b/apps/desktop/stories/session-workbar.stories.tsx index 213f83d284..43d14bb21b 100644 --- a/apps/desktop/stories/session-workbar.stories.tsx +++ b/apps/desktop/stories/session-workbar.stories.tsx @@ -705,6 +705,7 @@ const emptyUsageSummary: WorkbarSessionUsageSummary = { range: { from: NOW, to: NOW }, totalRequests: 0, totalCostUsd: 0, + totalDurationMs: 0, totalTokens: { input: 0, output: 0, @@ -736,6 +737,7 @@ const populatedUsageSummary: WorkbarSessionUsageSummary = { range: { from: NOW, to: NOW + 43_600 }, totalRequests: 3, totalCostUsd: 0.0243, + totalDurationMs: 38_400, totalTokens: { input: 81_300, output: 740, diff --git a/packages/core/src/__tests__/model-call-usage-projection.test.ts b/packages/core/src/__tests__/model-call-usage-projection.test.ts index 89a868f171..c8e57ec581 100644 --- a/packages/core/src/__tests__/model-call-usage-projection.test.ts +++ b/packages/core/src/__tests__/model-call-usage-projection.test.ts @@ -106,6 +106,19 @@ describe('model-call usage projection', () => { assert.equal(summary.coverage.unpricedAttempts, 1); }); + test('the summary sums recorded call time over the rows it counts', () => { + const summary = projectModelCallUsageSummary( + [ + attempt({ attemptId: 'a', logicalCallId: 'a', latencyMs: 1_200 }), + attempt({ attemptId: 'b', logicalCallId: 'b', latencyMs: 300 }), + ], + { range: 'all' }, + NOW, + ); + assert.equal(summary.totalDurationMs, 1_500); + assert.equal(summary.totalRequests, 2); + }); + test('a genuinely free call is still counted as priced', () => { const summary = projectModelCallUsageSummary( [attempt({ attemptId: 'free', costUsd: 0 })], diff --git a/packages/core/src/__tests__/usage-ledger-merge.test.ts b/packages/core/src/__tests__/usage-ledger-merge.test.ts index 97457402d7..9f031d064f 100644 --- a/packages/core/src/__tests__/usage-ledger-merge.test.ts +++ b/packages/core/src/__tests__/usage-ledger-merge.test.ts @@ -81,6 +81,7 @@ function legacySummary(overrides: Partial = {}): UsageSummaryV2 cacheHitRequests: 0, cacheCreateRequests: 0, errorRequests: 1, + totalDurationMs: 0, ...overrides, }; } @@ -144,6 +145,34 @@ describe('usage ledger merge', () => { assert.equal(merged.provenance.coverage.pricedAttempts, 1); }); + test('merges recorded call time from both ledgers', () => { + const merged = mergeUsageSummary( + legacySummary({ totalDurationMs: 700 }), + { + attempts: [attempt({ attemptId: 'a', latencyMs: 500 })], + unreadableRecords: 0, + pendingRepairs: 0, + }, + { range: 'all' }, + NOW, + ); + assert.equal(merged.totalDurationMs, 1_200); + + // The projection always measures the attempts it counts; a legacy store + // with no recorded time simply contributes a zero to the sum. + const canonicalOnly = mergeUsageSummary( + legacySummary(), + { + attempts: [attempt({ attemptId: 'a', latencyMs: 500 })], + unreadableRecords: 0, + pendingRepairs: 0, + }, + { range: 'all' }, + NOW, + ); + assert.equal(canonicalOnly.totalDurationMs, 500); + }); + test('unpriced canonical spend stays out of the total and is reported instead', () => { const merged = mergeUsageSummary( legacySummary({ totalRequests: 0, totalCostUsd: 0, errorRequests: 0 }), diff --git a/packages/core/src/model-call-usage-projection.ts b/packages/core/src/model-call-usage-projection.ts index 74c9b84194..3c394edbe8 100644 --- a/packages/core/src/model-call-usage-projection.ts +++ b/packages/core/src/model-call-usage-projection.ts @@ -48,6 +48,8 @@ import type { * its coverage repeats that claim. */ export interface ModelCallUsageSummary extends UsageSummaryV2 { + /** Always present: the projection measures every attempt it counts. */ + totalDurationMs: number; coverage: ModelCallCoverage; } @@ -167,6 +169,7 @@ export function projectModelCallUsageSummary( total: 0, }; let totalCostUsd = 0; + let totalDurationMs = 0; let cacheHitRequests = 0; let cacheCreateRequests = 0; let errorRequests = 0; @@ -180,6 +183,7 @@ export function projectModelCallUsageSummary( totals.reasoning += t.reasoning; totals.total += t.total; totalCostUsd += pricedCost(attempt); + totalDurationMs += attempt.latencyMs; if (t.cacheRead > 0) cacheHitRequests += 1; if (t.cacheWrite > 0) cacheCreateRequests += 1; if (usageStatusForAttempt(attempt.status) === 'error') errorRequests += 1; @@ -188,6 +192,7 @@ export function projectModelCallUsageSummary( range, totalRequests: rows.length, totalCostUsd, + totalDurationMs, totalTokens: totals, cacheHitRequests, cacheCreateRequests, diff --git a/packages/core/src/usage-ledger-merge.ts b/packages/core/src/usage-ledger-merge.ts index 2d715ac26e..f176ea0e5f 100644 --- a/packages/core/src/usage-ledger-merge.ts +++ b/packages/core/src/usage-ledger-merge.ts @@ -158,6 +158,9 @@ export function mergeUsageSummary( range: projected.range, totalRequests: legacy.totalRequests + projected.totalRequests, totalCostUsd: legacy.totalCostUsd + projected.totalCostUsd, + // The projection always measures the attempts it counted, and every legacy + // summary carries the same field. + totalDurationMs: legacy.totalDurationMs + projected.totalDurationMs, totalTokens: { input: legacy.totalTokens.input + projected.totalTokens.input, output: legacy.totalTokens.output + projected.totalTokens.output, diff --git a/packages/core/src/usage-stats/types.ts b/packages/core/src/usage-stats/types.ts index 53a6864eab..4b3f1a13bb 100644 --- a/packages/core/src/usage-stats/types.ts +++ b/packages/core/src/usage-stats/types.ts @@ -63,6 +63,23 @@ export interface UsageSummaryV2 { cacheHitRequests: number; cacheCreateRequests: number; errorRequests: number; + /** + * Recorded model-call time, summed over the same rows as `totalTokens`. + * + * Every summary writes it, and the protocol epoch refuses peers that predate + * it at the handshake, so a total without a time basis cannot cross the wire. + */ + totalDurationMs: number; + /** + * Recorded tool executions behind the same query, from the tool-invocation + * ledger — the model-call ledger does not describe them, so there is no + * canonical source to merge and this comes from the store alone. + * + * Optional because tool rows that predate connection attribution cannot + * answer a `connectionSlug` filter honestly: the Host omits the split for + * that query rather than drawing a ring from rows it cannot scope. + */ + toolUsage?: { requests: number; durationMs: number }; } export interface UsageBucket { diff --git a/packages/runtime-host/src/__tests__/usage-pricing-protocol.test.ts b/packages/runtime-host/src/__tests__/usage-pricing-protocol.test.ts index bed834df9a..2f7efffb7c 100644 --- a/packages/runtime-host/src/__tests__/usage-pricing-protocol.test.ts +++ b/packages/runtime-host/src/__tests__/usage-pricing-protocol.test.ts @@ -330,6 +330,66 @@ describe('Usage/Pricing protocol', () => { } }); + test('a summary always carries its recorded time; only the tool split is optional', () => { + // The epoch bump makes the duration basis a handshake requirement, so a + // summary without one is not an older host — it is a malformed frame. + assert.doesNotThrow(() => + usageResponse({ + kind: 'summary', + summary: { + ...validSummary(), + totalDurationMs: 1_500, + toolUsage: { requests: 3, durationMs: 450 }, + }, + provenance: validProvenance(), + }), + ); + assert.doesNotThrow(() => + usageResponse({ + kind: 'summary', + summary: validSummary(), + provenance: validProvenance(), + }), + ); + assert.throws( + () => + usageResponse({ + kind: 'summary', + summary: { ...validSummary(), totalDurationMs: undefined }, + provenance: validProvenance(), + }), + invalidFrame, + ); + assert.throws( + () => + usageResponse({ + kind: 'summary', + summary: { ...validSummary(), toolUsage: { requests: 3 } }, + provenance: validProvenance(), + }), + invalidFrame, + ); + assert.throws( + () => + usageResponse({ + kind: 'summary', + summary: { ...validSummary(), totalDurationMs: -1 }, + provenance: validProvenance(), + }), + invalidFrame, + ); + // An unknown key is still an unknown key, optional or not. + assert.throws( + () => + usageResponse({ + kind: 'summary', + summary: { ...validSummary(), totalWallClockMs: 1 }, + provenance: validProvenance(), + }), + invalidFrame, + ); + }); + test('keeps long usage identities distinct through the real coordinator and protocol', async () => { const base = await mkdtemp(join(tmpdir(), 'maka-usage-identity-projection-')); const capability = await resolveStorageRoot({ @@ -733,6 +793,7 @@ function validSummary() { cacheHitRequests: 0, cacheCreateRequests: 0, errorRequests: 0, + totalDurationMs: 1_200, }; } diff --git a/packages/runtime-host/src/__tests__/usage-pricing-two-client-uds.test.ts b/packages/runtime-host/src/__tests__/usage-pricing-two-client-uds.test.ts index afd2d44c48..83abf10185 100644 --- a/packages/runtime-host/src/__tests__/usage-pricing-two-client-uds.test.ts +++ b/packages/runtime-host/src/__tests__/usage-pricing-two-client-uds.test.ts @@ -367,6 +367,45 @@ test('usage logs carry the Host-resolved session title and tolerate unreadable s }); }); +test('a connection-scoped summary omits the tool split instead of sending an unscoped one', async () => { + await withUsageAuthority('summary-slug-omission', async ({ stores }) => { + await Promise.all([ + stores.telemetry.recordLlmCall({ + ...usageRecord('llm-slug', 30, 'openai', 'gpt-a'), + connectionSlug: 'openai', + }), + stores.telemetry.recordToolInvocation(toolRecord('tool-slug', 31)), + ]); + const coordinator = new HostUsagePricingCoordinator( + stores, + () => {}, + new RuntimePolicyActivationGate(), + () => {}, + async () => undefined, + ); + + // Tool rows predate connection attribution, so a connectionSlug-filtered + // query cannot scope them; the summary omits the split rather than let the + // tool ring quietly contradict the model totals beside it. + const scoped = await coordinator.handlers['usage.query']( + { kind: 'summary', query: { range: 'all', connectionSlug: 'openai' } }, + CONNECTION_CONTEXT, + ); + assert.ok(scoped.ok); + if (scoped.result.kind !== 'summary') throw new Error('expected summary'); + assert.equal(scoped.result.summary.toolUsage, undefined); + assert.equal(scoped.result.summary.totalRequests, 1); + + const unscoped = await coordinator.handlers['usage.query']( + { kind: 'summary', query: { range: 'all' } }, + CONNECTION_CONTEXT, + ); + assert.ok(unscoped.ok); + if (unscoped.result.kind !== 'summary') throw new Error('expected summary'); + assert.deepEqual(unscoped.result.summary.toolUsage, { requests: 1, durationMs: 12 }); + }); +}); + test('a non–not-found title read failure propagates out of usage.query instead of blanking the row', async () => { await withUsageAuthority('session-title-failure', async ({ stores }) => { await stores.telemetry.recordLlmCall({ diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index 90e7197a60..9abf64b3a6 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -100,7 +100,10 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 104 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 105 as const; +// 105: Usage summaries may carry the recorded call-time total and per-Session +// tool-invocation totals. Older Clients reject the unknown fields, so a newer +// Host's usage summary is unreadable to them. // 104: WorkHub Coordination actions add closed direct-stop proposals, // confirmations, expected-state preconditions, and outcomes. Older peers // reject these strict shapes. diff --git a/packages/runtime-host/src/protocol/usage-pricing.ts b/packages/runtime-host/src/protocol/usage-pricing.ts index 007bcda93d..22e6f72ee7 100644 --- a/packages/runtime-host/src/protocol/usage-pricing.ts +++ b/packages/runtime-host/src/protocol/usage-pricing.ts @@ -816,15 +816,22 @@ function decodeUsagePagePosition( } function decodeUsageSummary(value: unknown): UsageSummaryV2 { - const summary = requireExactRecord(value, 'usage summary', [ - 'range', - 'totalRequests', - 'totalCostUsd', - 'totalTokens', - 'cacheHitRequests', - 'cacheCreateRequests', - 'errorRequests', - ]); + const summary = requireRecord(value, 'usage summary'); + assertOptionalExactKeys( + summary, + 'usage summary', + [ + 'range', + 'totalRequests', + 'totalCostUsd', + 'totalTokens', + 'cacheHitRequests', + 'cacheCreateRequests', + 'errorRequests', + 'totalDurationMs', + ], + ['toolUsage'], + ); const range = requireExactRecord(summary.range, 'usage summary range', ['from', 'to']); const tokens = requireExactRecord(summary.totalTokens, 'usage summary tokens', [ 'input', @@ -854,6 +861,19 @@ function decodeUsageSummary(value: unknown): UsageSummaryV2 { cacheHitRequests: requireCount(summary.cacheHitRequests, 'usage cache hit requests'), cacheCreateRequests: requireCount(summary.cacheCreateRequests, 'usage cache create requests'), errorRequests: requireCount(summary.errorRequests, 'usage error requests'), + totalDurationMs: requireCount(summary.totalDurationMs, 'usage total duration'), + // Optional for a data reason, not a version one: tool rows that predate + // connection attribution cannot answer a `connectionSlug` filter, so the + // Host omits the split for that query rather than send an unscoped total. + ...(summary.toolUsage !== undefined ? { toolUsage: decodeToolUsage(summary.toolUsage) } : {}), + }; +} + +function decodeToolUsage(value: unknown): NonNullable { + const toolUsage = requireExactRecord(value, 'usage tool usage', ['requests', 'durationMs']); + return { + requests: requireCount(toolUsage.requests, 'usage tool requests'), + durationMs: requireCount(toolUsage.durationMs, 'usage tool duration'), }; } diff --git a/packages/runtime-host/src/server/usage-pricing-coordinator.ts b/packages/runtime-host/src/server/usage-pricing-coordinator.ts index 1404664657..ab53dd0e3b 100644 --- a/packages/runtime-host/src/server/usage-pricing-coordinator.ts +++ b/packages/runtime-host/src/server/usage-pricing-coordinator.ts @@ -147,10 +147,25 @@ export class HostUsagePricingCoordinator { input.query, now, ); + // Tool executions are in their own ledger, not the model-call one, so + // their totals ride beside the merged summary rather than inside it — + // the same owner split the tool buckets path already follows. A + // connection-scoped query is refused instead of answered: tool rows + // that predate connection attribution cannot be scoped, and a ring + // built from an unscoped subset would quietly contradict the model + // totals beside it. const { provenance, ...summary } = merged; + const toolUsage = + input.query.connectionSlug === undefined + ? await this.#stores.telemetry.toolSummary(input.query) + : undefined; return { ok: true, - result: encodeUsageQueryResult({ kind: 'summary', summary, provenance }), + result: encodeUsageQueryResult({ + kind: 'summary', + summary: { ...summary, toolUsage }, + provenance, + }), }; } if (input.kind === 'buckets') { diff --git a/packages/storage/src/__tests__/usage-stores.test.ts b/packages/storage/src/__tests__/usage-stores.test.ts index c00e6f0b5b..f8b1512538 100644 --- a/packages/storage/src/__tests__/usage-stores.test.ts +++ b/packages/storage/src/__tests__/usage-stores.test.ts @@ -174,6 +174,29 @@ describe('InteractiveUsageStores', () => { }); }); + test('publishes the owning Session after a durable tool-usage write', async () => { + await withInteractiveRoot(async ({ capability }) => { + const owner = await tryAcquireInteractiveRootOwner(capability); + assert(owner); + const stores = await openInteractiveUsageStoresForWrite(owner.lease); + const changed: string[] = []; + const unsubscribe = stores.subscribeSessionUsageChanges((sessionId) => + changed.push(sessionId), + ); + try { + // A session whose last activity is a tool invocation must still see the + // usage summary refresh — the trace panel's time ring reads the + // summary on exactly this signal. + await stores.telemetry.recordToolInvocation(toolRecord({ sessionId: 'session-tool' })); + assert.deepEqual(changed, ['session-tool']); + } finally { + unsubscribe(); + await stores.close(); + await owner.close(); + } + }); + }); + test('does not republish idempotent model-usage mutations', async () => { await withInteractiveRoot(async ({ root, capability }) => { const owner = await tryAcquireInteractiveRootOwner(capability); @@ -416,6 +439,162 @@ describe('InteractiveUsageStores', () => { }); }); + test('legacy summary sums recorded call time over the same rows as its tokens', async () => { + await withInteractiveRoot(async ({ capability }) => { + const owner = await tryAcquireInteractiveRootOwner(capability); + assert(owner); + const stores = await openInteractiveUsageStoresForWrite(owner.lease); + await stores.telemetry.recordLlmCall(llmRecord({ id: 'call-a', latencyMs: 1_200 })); + await stores.telemetry.recordLlmCall(llmRecord({ id: 'call-b', latencyMs: 300 })); + + const summary = await stores.telemetry.summary({ range: 'all' }); + assert.equal(summary.totalDurationMs, 1_500); + + await stores.close(); + await owner.close(); + }); + }); + + test('tool summary scopes to the requested Session and range', async () => { + await withInteractiveRoot(async ({ capability }) => { + const owner = await tryAcquireInteractiveRootOwner(capability); + assert(owner); + const stores = await openInteractiveUsageStoresForWrite(owner.lease); + await stores.telemetry.recordToolInvocation( + toolRecord({ id: 'tool-a', sessionId: 'session-a', durationMs: 120 }), + ); + await stores.telemetry.recordToolInvocation( + toolRecord({ id: 'tool-b', sessionId: 'session-b', durationMs: 80 }), + ); + + const sessionA = await stores.telemetry.toolSummary({ + range: 'all', + sessionId: 'session-a', + }); + assert.deepEqual(sessionA, { requests: 1, durationMs: 120 }); + + // Without a session filter the ledger answers with everything in range — + // the same contract the tool buckets follow. + const everySession = await stores.telemetry.toolSummary({ range: 'all' }); + assert.deepEqual(everySession, { requests: 2, durationMs: 200 }); + + const empty = await stores.telemetry.toolSummary({ + range: { from: 0, to: 1 }, + sessionId: 'session-a', + }); + assert.deepEqual(empty, { requests: 0, durationMs: 0 }); + + await stores.close(); + await owner.close(); + }); + }); + + test('tool summary applies the full summary query to the tool rows', async () => { + await withInteractiveRoot(async ({ capability }) => { + const owner = await tryAcquireInteractiveRootOwner(capability); + assert(owner); + const stores = await openInteractiveUsageStoresForWrite(owner.lease); + await stores.telemetry.recordToolInvocation( + toolRecord({ + id: 'tool-openai-ok', + sessionId: 'session-a', + toolName: 'Bash', + providerId: 'openai', + modelId: 'gpt-5', + status: 'success', + durationMs: 100, + }), + ); + await stores.telemetry.recordToolInvocation( + toolRecord({ + id: 'tool-anthropic-err', + sessionId: 'session-a', + toolName: 'Read', + providerId: 'anthropic', + modelId: 'claude-opus-5', + status: 'error', + durationMs: 300, + }), + ); + + // The tool ring sits beside the model totals under one query, so a + // filter the rows can answer must narrow both sides the same way. + const provider = await stores.telemetry.toolSummary({ + range: 'all', + sessionId: 'session-a', + providerId: 'openai', + }); + assert.deepEqual(provider, { requests: 1, durationMs: 100 }); + + const status = await stores.telemetry.toolSummary({ + range: 'all', + sessionId: 'session-a', + status: 'error', + }); + assert.deepEqual(status, { requests: 1, durationMs: 300 }); + + const model = await stores.telemetry.toolSummary({ + range: 'all', + sessionId: 'session-a', + modelId: 'gpt-5', + }); + assert.deepEqual(model, { requests: 1, durationMs: 100 }); + + const tool = await stores.telemetry.toolSummary({ + range: 'all', + sessionId: 'session-a', + toolName: 'Read', + }); + assert.deepEqual(tool, { requests: 1, durationMs: 300 }); + + await stores.close(); + await owner.close(); + }); + }); + + test('tool buckets answer the full summary query, including Session and provider', async () => { + await withInteractiveRoot(async ({ capability }) => { + const owner = await tryAcquireInteractiveRootOwner(capability); + assert(owner); + const stores = await openInteractiveUsageStoresForWrite(owner.lease); + // Two rows share provider and model, so only the session (and the tool + // name) can tell them apart — the bucket view must apply the same + // filters the summary beside it applies. + await stores.telemetry.recordToolInvocation( + toolRecord({ + id: 'bucket-session-a', + sessionId: 'session-a', + toolName: 'Bash', + providerId: 'openai', + modelId: 'gpt-5', + durationMs: 100, + }), + ); + await stores.telemetry.recordToolInvocation( + toolRecord({ + id: 'bucket-session-b', + sessionId: 'session-b', + toolName: 'Bash', + providerId: 'openai', + modelId: 'gpt-5', + durationMs: 400, + }), + ); + + const scoped = await stores.telemetry.buckets( + { range: 'all', sessionId: 'session-a', providerId: 'openai' }, + 'tool', + ); + assert.deepEqual( + scoped.map((bucket) => [bucket.key, bucket.requests, bucket.avgLatencyMs]), + [['Bash', 1, 100]], + ); + + await stores.close(); + await owner.close(); + }); + }); + test('every facade read observes lease revocation', async () => { await withInteractiveRoot(async ({ capability }) => { const owner = await tryAcquireInteractiveRootOwner(capability); @@ -502,7 +681,7 @@ function llmRecord(overrides: Record = {}) { >[0]; } -function toolRecord() { +function toolRecord(overrides: Record = {}) { return { id: 'tool_1', toolName: 'Bash', @@ -513,6 +692,7 @@ function toolRecord() { date: '2026-01-01', ts: Date.UTC(2026, 0, 1), startedAt: Date.UTC(2026, 0, 1), + ...overrides, } as Parameters< Awaited< ReturnType diff --git a/packages/storage/src/sqlite-usage-store.ts b/packages/storage/src/sqlite-usage-store.ts index 2b73e859d3..2cb66c03e3 100644 --- a/packages/storage/src/sqlite-usage-store.ts +++ b/packages/storage/src/sqlite-usage-store.ts @@ -179,6 +179,7 @@ class SqliteTelemetryRepo implements TelemetryRepo { range: { from, to }, totalRequests: rows.length, totalCostUsd: sum(rows.map((row) => row.costUsd)), + totalDurationMs: sum(rows.map((row) => row.latencyMs)), totalTokens: { input: sum(rows.map((row) => row.inputTokens)), output: sum(rows.map((row) => row.outputTokens)), @@ -198,6 +199,16 @@ class SqliteTelemetryRepo implements TelemetryRepo { }); } + toolSummary(query: UsageQuery): { requests: number; durationMs: number } { + this.assertReady(); + const { from, to } = resolveRange(query.range); + const rows = this.filteredToolRows(query, from, to); + return detached({ + requests: rows.length, + durationMs: sum(rows.map((row) => row.durationMs)), + }); + } + buckets(query: UsageQuery, groupBy: UsageGroupBy): UsageBucket[] { this.assertReady(); const { from, to } = resolveRange(query.range); @@ -300,10 +311,19 @@ class SqliteTelemetryRepo implements TelemetryRepo { } private filteredToolRows(query: UsageQuery | ToolUsageQuery, from: number, to: number) { - return this.readToolRows().filter((row) => { - if (row.ts < from || row.ts > to) return false; + // One filter policy for every tool read — summary, buckets, and logs — so + // two views of one query cannot disagree. Fields the narrower query types + // do not carry simply never match a row out of range. + return this.readToolRows(from, to).filter((row) => { if (query.toolName && row.toolName !== query.toolName) return false; if (query.status && query.status !== 'all' && row.status !== query.status) return false; + if ('sessionId' in query && query.sessionId && row.sessionId !== query.sessionId) { + return false; + } + if ('providerId' in query && query.providerId && row.providerId !== query.providerId) { + return false; + } + if ('modelId' in query && query.modelId && row.modelId !== query.modelId) return false; return true; }); } @@ -324,11 +344,15 @@ class SqliteTelemetryRepo implements TelemetryRepo { ).map((row) => decodePersistedLlmCallRecord(JSON.parse(row.record_json))); } - private readToolRows(): PersistedToolInvocationRecord[] { + // The ts range goes into the query, not the decode: the invocation ledger + // has no retention, and a full-table decode per summary read would only grow + // with it. The `(ts, id)` index answers the range; the remaining filters run + // over the few decoded rows it returns. + private readToolRows(from: number, to: number): PersistedToolInvocationRecord[] { return ( this.#lease.database - .prepare('SELECT record_json FROM usage_tool_invocations') - .all() as Array<{ record_json: string }> + .prepare('SELECT record_json FROM usage_tool_invocations WHERE ts >= ? AND ts <= ?') + .all(from, to) as Array<{ record_json: string }> ).map((row) => decodePersistedToolInvocationRecord(JSON.parse(row.record_json))); } diff --git a/packages/storage/src/telemetry-repo.ts b/packages/storage/src/telemetry-repo.ts index c3ed654452..285e04413a 100644 --- a/packages/storage/src/telemetry-repo.ts +++ b/packages/storage/src/telemetry-repo.ts @@ -45,6 +45,12 @@ export interface TelemetryRepo { insertLlmCall(record: PersistedLlmCallRecord): Promise; insertToolInvocation(record: PersistedToolInvocationRecord): Promise; summary(query: UsageQuery): UsageSummaryV2; + /** + * Session- and range-scoped totals over the tool-invocation ledger — the one + * aggregate the LLM summary cannot answer, because tool executions live in + * their own table with no canonical counterpart to merge. + */ + toolSummary(query: UsageQuery): { requests: number; durationMs: number }; buckets(query: UsageQuery, groupBy: UsageGroupBy): UsageBucket[]; logs(query: UsageQuery, offset?: number, limit?: number): { rows: UsageLogRow[]; total: number }; toolLogs( diff --git a/packages/storage/src/usage-stores.ts b/packages/storage/src/usage-stores.ts index 19823028e3..3e4b3a8953 100644 --- a/packages/storage/src/usage-stores.ts +++ b/packages/storage/src/usage-stores.ts @@ -74,6 +74,7 @@ const writerOpeningByLease = new WeakMap; + toolSummary(query: UsageQuery): Promise<{ requests: number; durationMs: number }>; buckets(query: UsageQuery, groupBy: UsageGroupBy): Promise; logs( query: UsageQuery, @@ -454,6 +455,7 @@ function createWriterFacade( [writerBrand]: true, telemetry: { summary: (query) => read(() => telemetry.summary(query)), + toolSummary: (query) => read(() => telemetry.toolSummary(query)), buckets: (query, groupBy) => read(() => telemetry.buckets(query, groupBy)), logs: (query, offset, limit) => read(() => telemetry.logs(query, offset, limit)), toolLogs: (query, offset, limit) => read(() => telemetry.toolLogs(query, offset, limit)), @@ -462,7 +464,7 @@ function createWriterFacade( recordLlmCall: (record) => admitSessionUsageMutation(record.sessionId, () => telemetry.insertLlmCall(record)), recordToolInvocation: (record) => - admit(() => run(() => telemetry.insertToolInvocation(record))), + admitSessionUsageMutation(record.sessionId, () => telemetry.insertToolInvocation(record)), }, modelCalls: { modelCallAttempts: (range, sessionId) => read(() => modelCalls.read(range, sessionId)), @@ -497,6 +499,7 @@ function telemetryReader( ): Readonly { return Object.freeze({ summary: (query: UsageQuery) => run(() => repo.summary(query)), + toolSummary: (query: UsageQuery) => run(() => repo.toolSummary(query)), buckets: (query: UsageQuery, groupBy: UsageGroupBy) => run(() => repo.buckets(query, groupBy)), logs: (query: UsageQuery, offset?: number, limit?: number) => run(() => repo.logs(query, offset, limit)),