diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index fb29abb545..a120758a10 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -930,6 +930,8 @@ export interface TurnStateMessage { /** Diagnostic source for user/renderer-triggered aborts, e.g. renderer.stop_button. */ abortSource?: string; errorClass?: string; + /** Bounded provider response summary for a failed turn. */ + failureMessage?: string; partialOutputRetained: boolean; } @@ -1056,6 +1058,8 @@ export interface TurnRecord { abortedAt?: number; abortSource?: string; errorClass?: string; + /** Bounded provider response summary for a failed turn. */ + failureMessage?: string; partialOutputRetained: boolean; } @@ -1163,6 +1167,7 @@ const TURN_STATE_MESSAGE_SHAPE = defineObjectShape()( 'abortedAt', 'abortSource', 'errorClass', + 'failureMessage', ], ); const WORKHUB_DELEGATION_ASSIGNED_MESSAGE_SHAPE = @@ -1409,7 +1414,8 @@ function decodeMessage( isOptionalString(message.parentSessionId) && (message.abortedAt === undefined || isFiniteNumber(message.abortedAt)) && isOptionalString(message.abortSource) && - isOptionalString(message.errorClass) + isOptionalString(message.errorClass) && + isOptionalString(message.failureMessage) ) return message as unknown as TurnStateMessage; break; @@ -1656,6 +1662,7 @@ export function deriveTurnRecords(messages: readonly StoredMessage[]): TurnRecor ...(latestState.abortedAt !== undefined ? { abortedAt: latestState.abortedAt } : {}), ...(latestState.abortSource ? { abortSource: latestState.abortSource } : {}), ...(latestState.errorClass ? { errorClass: latestState.errorClass } : {}), + ...(latestState.failureMessage ? { failureMessage: latestState.failureMessage } : {}), partialOutputRetained: latestState.partialOutputRetained || partialOutputRetained, }; } diff --git a/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts b/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts index 06eff1cae3..9882440630 100644 --- a/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts +++ b/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts @@ -380,6 +380,9 @@ test('projects a failed Turn message from the canonical terminal event', async ( recoverable: false, code: 'provider_error', message: 'canonical provider failure api_key=sk-test-secret-value', + details: { + providerSummary: 'provider rejected model (code=provider_error, requestId=req-123)', + }, }, context, memory, @@ -417,7 +420,7 @@ test('projects a failed Turn message from the canonical terminal event', async ( if (canonical?.rootTurn?.status === 'failed') { assert.equal( canonical.rootTurn.failureMessage, - 'canonical provider failure api_key=[redacted]', + 'provider rejected model (code=provider_error, requestId=req-123)', ); } }); diff --git a/packages/runtime-host/src/__tests__/session-turns.test.ts b/packages/runtime-host/src/__tests__/session-turns.test.ts index 945e36e780..a09b30f2e7 100644 --- a/packages/runtime-host/src/__tests__/session-turns.test.ts +++ b/packages/runtime-host/src/__tests__/session-turns.test.ts @@ -26,6 +26,7 @@ import { projectSessionTurnContribution, projectSessionTurnContributionForWire, SESSION_TURN_DIAGNOSTIC_MAX_BYTES, + SESSION_TURN_FAILURE_MESSAGE_MAX_BYTES, SESSION_TURN_LANDMARK_RESULT_MAX_BYTES, } from '../protocol/session-turns.js'; @@ -110,6 +111,47 @@ test('bounds turn diagnostics before publishing a contribution', () => { ); }); +test('persists and bounds the provider failure summary in the Turn projection', () => { + const contribution = projectSessionTurnContributionForWire({ + turnId: 'turn-1', + firstSequence: 0, + latestState: { + sequence: 0, + message: { + type: 'turn_state', + id: 'state-1', + turnId: 'turn-1', + ts: 1, + status: 'failed', + partialOutputRetained: false, + errorClass: 'rate_limit', + failureMessage: `provider says ${'x'.repeat(10_000)}`, + }, + }, + userPromptPreview: null, + hasAssistantMessage: false, + hasAssistantOutput: false, + hasToolResult: false, + hasFailedToolResult: false, + hasAbortNote: false, + }); + + assert.ok( + Buffer.byteLength(contribution.latestState!.message.failureMessage!, 'utf8') <= + SESSION_TURN_FAILURE_MESSAGE_MAX_BYTES, + ); + const decoded = decodeSessionTurnsQueryResult({ + sessionId: 'session-1', + throughSequence: 0, + contributions: [contribution], + nextPosition: null, + }); + assert.equal( + decoded.contributions[0]!.latestState!.message.failureMessage, + contribution.latestState!.message.failureMessage, + ); +}); + test('rejects invalid turn-state references before publishing a contribution', () => { assert.throws(() => projectSessionTurnContributionForWire({ diff --git a/packages/runtime-host/src/adapter/session-projector.ts b/packages/runtime-host/src/adapter/session-projector.ts index 7924fa38ad..2fe8e3ba9e 100644 --- a/packages/runtime-host/src/adapter/session-projector.ts +++ b/packages/runtime-host/src/adapter/session-projector.ts @@ -290,7 +290,10 @@ export class RuntimeHostSessionProjector { ts: terminal.ts, recoverable: false, reason, - message: `Turn failed: ${reason}`, + message: terminal.failureMessage ?? `Turn failed: ${reason}`, + ...(terminal.failureMessage + ? { details: { providerSummary: terminal.failureMessage } } + : {}), }); } else { events.push({ @@ -329,7 +332,8 @@ export class RuntimeHostSessionProjector { ts, recoverable: false, reason, - message: `Turn failed: ${reason}`, + message: turn.failureMessage ?? `Turn failed: ${reason}`, + ...(turn.failureMessage ? { details: { providerSummary: turn.failureMessage } } : {}), }, ]; } @@ -483,6 +487,7 @@ export class RuntimeHostSessionProjector { recoverable: false, reason: root.failureClass, message: root.failureMessage ?? `Turn failed: ${root.failureClass}`, + ...(root.failureMessage ? { details: { providerSummary: root.failureMessage } } : {}), }); } else { events.push({ diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index abac95994a..cb0c4738be 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -100,7 +100,10 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 101 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 102 as const; +// 102: Session Turn projections carry bounded provider failure summaries for +// live and reloaded failed-turn diagnostics. Older peers cannot preserve or +// render this additional failure context safely. // 101: Session Turn requests can carry regeneration intents and Guests can // atomically withdraw pending requests. Older peers do not share this command // vocabulary or the expanded Guest operation grant. diff --git a/packages/runtime-host/src/protocol/session-turns.ts b/packages/runtime-host/src/protocol/session-turns.ts index c615b7152a..b3c0df009d 100644 --- a/packages/runtime-host/src/protocol/session-turns.ts +++ b/packages/runtime-host/src/protocol/session-turns.ts @@ -32,6 +32,7 @@ import { defineOperation } from './operation-spec.js'; export const SESSION_TURN_QUERY_MAX_CONTRIBUTIONS = 128; export const SESSION_TURN_QUERY_RESULT_MAX_BYTES = 192 * 1024; export const SESSION_TURN_DIAGNOSTIC_MAX_BYTES = 128; +export const SESSION_TURN_FAILURE_MESSAGE_MAX_BYTES = 256; export const SESSION_TURN_PROMPT_PREVIEW_MAX_BYTES = 256; export const SESSION_TURN_LANDMARK_MAX_ITEMS = 64; export const SESSION_TURN_LANDMARK_LABEL_MAX_BYTES = 96; @@ -179,6 +180,14 @@ function projectTurnStateMessageForWire(message: TurnStateMessage): TurnStateMes ...(message.errorClass ? { errorClass: truncateUtf8(message.errorClass, SESSION_TURN_DIAGNOSTIC_MAX_BYTES) } : {}), + ...(message.failureMessage + ? { + failureMessage: truncateUtf8( + message.failureMessage, + SESSION_TURN_FAILURE_MESSAGE_MAX_BYTES, + ), + } + : {}), partialOutputRetained: message.partialOutputRetained, }; } @@ -205,6 +214,7 @@ export function projectSessionTurnContribution(contribution: SessionTurnContribu ...(state.abortedAt !== undefined ? { abortedAt: state.abortedAt } : {}), ...(state.abortSource ? { abortSource: state.abortSource } : {}), ...(state.errorClass ? { errorClass: state.errorClass } : {}), + ...(state.failureMessage ? { failureMessage: state.failureMessage } : {}), partialOutputRetained: state.partialOutputRetained || partialOutputRetained, }; } @@ -424,6 +434,13 @@ function decodeSessionTurnContribution(value: unknown): SessionTurnContribution SESSION_TURN_DIAGNOSTIC_MAX_BYTES, ); } + if (message.failureMessage !== undefined) { + requireUtf8String( + message.failureMessage, + 'Session turn failure message', + SESSION_TURN_FAILURE_MESSAGE_MAX_BYTES, + ); + } latestState = { sequence: requireCount(state.sequence, 'Session turn state sequence'), message, diff --git a/packages/runtime-host/src/server/canonical-turn-snapshot.ts b/packages/runtime-host/src/server/canonical-turn-snapshot.ts index bdf7a3ae17..684c21a9ce 100644 --- a/packages/runtime-host/src/server/canonical-turn-snapshot.ts +++ b/packages/runtime-host/src/server/canonical-turn-snapshot.ts @@ -73,7 +73,10 @@ export async function readCanonicalTurnSnapshot( const failureMessage = fact.terminalEvent.content?.kind === 'error' ? truncateUtf8( - redactSecrets(fact.terminalEvent.content.message), + redactSecrets( + providerFailureSummaryFromRuntimeEvent(fact.terminalEvent) ?? + fact.terminalEvent.content.message, + ), TURN_FAILURE_MESSAGE_MAX_BYTES, '…', ) @@ -110,6 +113,14 @@ export async function readCanonicalTurnSnapshot( return { sessionId, turnId, runId, status: run.status }; } +function providerFailureSummaryFromRuntimeEvent( + event: import('@maka/core/runtime-event').RuntimeEvent, +): string | undefined { + const details = event.content?.kind === 'error' ? event.content.details : undefined; + if (!details || Array.isArray(details)) return undefined; + const summary = details.providerSummary; + return typeof summary === 'string' && summary.length > 0 ? summary : undefined; +} function readContextCompactionOutcome(value: unknown): ContextCompactionOutcome | undefined { if (!value || typeof value !== 'object') return undefined; const outcome = value as Record; diff --git a/packages/runtime-host/src/server/shared-session-transcript.ts b/packages/runtime-host/src/server/shared-session-transcript.ts index 5c59dd8b0e..b113d376fc 100644 --- a/packages/runtime-host/src/server/shared-session-transcript.ts +++ b/packages/runtime-host/src/server/shared-session-transcript.ts @@ -136,6 +136,7 @@ export function projectSharedSessionTranscriptMessage( ...(message.abortedAt === undefined ? {} : { abortedAt: message.abortedAt }), ...(message.abortSource === undefined ? {} : { abortSource: message.abortSource }), ...(message.errorClass === undefined ? {} : { errorClass: message.errorClass }), + ...(message.failureMessage === undefined ? {} : { failureMessage: message.failureMessage }), partialOutputRetained: message.partialOutputRetained, }; case 'token_usage': diff --git a/packages/runtime/src/__tests__/model-adapter-onerror.test.ts b/packages/runtime/src/__tests__/model-adapter-onerror.test.ts index c789c6548a..aa03e448da 100644 --- a/packages/runtime/src/__tests__/model-adapter-onerror.test.ts +++ b/packages/runtime/src/__tests__/model-adapter-onerror.test.ts @@ -154,6 +154,7 @@ describe('ModelAdapter.startStream onError', () => { message: 'Rate limit exceeded', retryable: true, retryAfterMs: 2500, + diagnosticSummary: 'rate limited (status=429)', }, ]); const outcome = await requireAlreadySettled(result.outcome); @@ -204,6 +205,7 @@ describe('ModelAdapter.startStream onError', () => { kind: 'unknown', message: 'Plaintext Responses reasoning item is missing final summary metadata', retryable: false, + diagnosticSummary: 'Plaintext Responses reasoning item is missing final summary metadata', }, ]); const outcome = await requireAlreadySettled(result.outcome); @@ -249,6 +251,7 @@ describe('ModelAdapter.startStream onError', () => { retryable: false, message: 'Rate limit exceeded', code: 'rate_limit_exceeded', + diagnosticSummary: 'Provider stopped the stream with an error (code=rate_limit_exceeded)', }); assert.equal(outcome.usage?.rawFinishReason, 'rate_limit_exceeded'); }); @@ -288,6 +291,7 @@ describe('ModelAdapter.startStream onError', () => { message: 'Rate limit exceeded', retryable: true, retryAfterMs: 2500, + diagnosticSummary: 'rate limited (status=429)', }, ]); assert.deepEqual(await result.outcome, { @@ -485,6 +489,8 @@ describe('ModelAdapter.startStream onError', () => { kind: 'network', message: 'Network error', retryable: true, + diagnosticSummary: + 'Client network socket disconnected before secure TLS connection was established', }, ]); } finally { diff --git a/packages/runtime/src/__tests__/model-adapter.test.ts b/packages/runtime/src/__tests__/model-adapter.test.ts index 3fa8751fe9..80ca297c7d 100644 --- a/packages/runtime/src/__tests__/model-adapter.test.ts +++ b/packages/runtime/src/__tests__/model-adapter.test.ts @@ -378,6 +378,7 @@ describe('ModelAdapter stream and error normalization', () => { code: '429', message: 'Rate limit exceeded', retryable: false, + diagnosticSummary: '429 rate limit (code=429)', }); // The backend consumes the typed failure without recovering the raw // provider error shape. @@ -385,6 +386,7 @@ describe('ModelAdapter stream and error normalization', () => { assert.equal(shaped.reason, 'rate_limit'); assert.equal(shaped.code, '429'); assert.equal(shaped.message, 'Rate limit exceeded'); + assert.deepEqual(shaped.details, { providerSummary: '429 rate limit (code=429)' }); }); test('normalizes a status-less provider server_error into a retryable outage', () => { @@ -408,6 +410,8 @@ describe('ModelAdapter stream and error normalization', () => { code: 'server_error', message: 'Provider returned an error', retryable: true, + diagnosticSummary: + 'Streaming response failed: [502] Upstream error from Nvidia: Service temporarily overloaded (code=server_error)', }, }); }); @@ -842,6 +846,24 @@ describe('ModelAdapter stream and error normalization', () => { assert.equal(event.message.includes('sk-live-secret-token-value'), false); }); + test('re-scrubs a pre-normalized provider diagnostic at the event boundary', () => { + const event = newAdapter().makeErrorEvent('turn-1', { + type: 'model_failure', + kind: 'rate_limit', + message: 'Rate limit exceeded', + retryable: false, + diagnosticSummary: `provider says api_key=sk-live-secret-token-value ${'x'.repeat(4_000)}`, + }); + + const providerSummary = + event.details && !Array.isArray(event.details) ? event.details.providerSummary : undefined; + assert.equal( + typeof providerSummary === 'string' && providerSummary.includes('sk-live-secret-token-value'), + false, + ); + assert.equal(Buffer.byteLength(String(providerSummary ?? ''), 'utf8') <= 2 * 1024, true); + }); + test('normalizes cache and reasoning usage variants in the adapter module', () => { assert.deepEqual( normalizeAiSdkUsage({ diff --git a/packages/runtime/src/__tests__/openai-responses-model-adapter.test.ts b/packages/runtime/src/__tests__/openai-responses-model-adapter.test.ts index 36052d5dd5..f760447cca 100644 --- a/packages/runtime/src/__tests__/openai-responses-model-adapter.test.ts +++ b/packages/runtime/src/__tests__/openai-responses-model-adapter.test.ts @@ -136,6 +136,8 @@ describe('OpenAI Responses ModelAdapter continuation', () => { retryable: true, code: 'OPENAI_RESPONSES_WEBSOCKET_TRANSPORT_ERROR', message: 'Network error', + diagnosticSummary: + 'closed before completion (code=OPENAI_RESPONSES_WEBSOCKET_TRANSPORT_ERROR)', }, ]); }); diff --git a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts index 73eb22437a..cbb36dfde6 100644 --- a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts +++ b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts @@ -1603,6 +1603,36 @@ describe('projectRuntimeEventsToStoredMessages', () => { assert.deepStrictEqual(out.diagnostics, []); }); + test('carries the bounded provider summary into the durable failed turn', () => { + const out = projectRuntimeEventsToStoredMessages( + [ + ev({ + id: 'evt-provider-failed', + ts: ts + 9, + status: 'failed', + content: { + kind: 'error', + reason: 'rate_limit', + message: 'Rate limit exceeded', + details: { + providerSummary: `provider says api_key=sk-live-secret ${'x'.repeat(1_000)}`, + }, + }, + actions: { endInvocation: true }, + }), + ], + { + runHeaders: [{ ...header, status: 'failed', failureClass: 'rate_limit' }], + }, + ); + + const state = out.messages.find((message) => message.type === 'turn_state'); + assert.equal(state?.type, 'turn_state'); + assert.equal(state?.errorClass, 'rate_limit'); + assert.equal(state?.failureMessage?.includes('sk-live-secret'), false); + assert.ok(Buffer.byteLength(state?.failureMessage ?? '', 'utf8') <= 256); + }); + test('tool step cap terminal fact projects a persistent system notice', () => { const out = projectRuntimeEventsToStoredMessages( [ diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index cff8aaed16..da8030c59a 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -76,6 +76,7 @@ import { buildStatusPatch, isTerminalRunStatus, normalizeStopSessionSource, + providerFailureMessageFromEvent, statusFromEvent, turnStatusFromEvent, } from './session-projection-helpers.js'; @@ -119,7 +120,12 @@ export interface AgentRunHooks { turnId: string, status: TurnRecord['status'], lineage?: AgentRunLineage, - options?: { ts?: number; errorClass?: string; abortSource?: string }, + options?: { + ts?: number; + errorClass?: string; + failureMessage?: string; + abortSource?: string; + }, ): Promise; } @@ -905,6 +911,7 @@ export class AgentRun { { ts: ev.ts, errorClass: turnStatus.errorClass, + failureMessage: turnStatus.failureMessage, ...(turnStatus.status === 'aborted' && this.abortSource ? { abortSource: this.abortSource } : {}), @@ -929,10 +936,15 @@ export class AgentRun { .appendTurnState(this.sessionId, this.turnId, 'failed', this.lineage, { ts: ev.ts, errorClass: ev.reason ?? ev.code ?? 'unknown', + failureMessage: ev.type === 'error' ? providerFailureMessageFromEvent(ev) : undefined, }) .catch((error) => this.enqueueTraceWriteFailure(error, 'terminal session projection')); - this.markRunFailed(ev.reason ?? ev.code ?? 'unknown', ev.message, ev.ts); + this.markRunFailed( + ev.reason ?? ev.code ?? 'unknown', + (ev.type === 'error' ? providerFailureMessageFromEvent(ev) : undefined) ?? ev.message, + ev.ts, + ); } } } diff --git a/packages/runtime/src/model-adapter.ts b/packages/runtime/src/model-adapter.ts index b5444bb529..b1e7a1a5d9 100644 --- a/packages/runtime/src/model-adapter.ts +++ b/packages/runtime/src/model-adapter.ts @@ -25,7 +25,8 @@ import { type RuntimeExecutionConnection, } from '@maka/core/llm-connections'; import { lookupModelMetadata } from '@maka/core/model-metadata'; -import { generalizedErrorMessage } from '@maka/core/redaction'; +import { generalizedErrorMessage, redactSecrets } from '@maka/core/redaction'; +import { truncateUtf8 } from '@maka/core/diagnostic-log'; import type { CacheMissInputSource } from '@maka/core/usage-stats/types'; import { rawFinishReasonString } from './model-protocol.js'; import type { @@ -63,6 +64,7 @@ import { import { classifyError, errorPresentationFromClass, + PROVIDER_FAILURE_SUMMARY_MAX_BYTES, providerFailureSummary, providerRetryMetadata, } from './provider-error-classification.js'; @@ -540,6 +542,14 @@ export class ModelAdapter { makeErrorEvent(turnId: string, err: unknown, reasonOverride?: string): ErrorEvent { const failure = normalizeModelFailure(err); + const diagnosticSummary = + isModelFailure(err) && typeof err.diagnosticSummary === 'string' + ? truncateUtf8( + redactSecrets(err.diagnosticSummary), + PROVIDER_FAILURE_SUMMARY_MAX_BYTES, + '…', + ) + : undefined; return { type: 'error', id: this.input.newId(), @@ -553,6 +563,7 @@ export class ModelAdapter { ? { reason: failure.kind } : {}), message: failure.message, + ...(diagnosticSummary ? { details: { providerSummary: diagnosticSummary } } : {}), }; } @@ -1269,6 +1280,7 @@ function normalizeProviderFailure(error: unknown): ModelFailure { return { ...failure, ...(summary?.code !== undefined ? { code: summary.code } : {}), + ...(summary?.message !== undefined ? { diagnosticSummary: summary.message } : {}), ...(failure.kind === 'unknown' && summary !== undefined ? { message: summary.message } : {}), }; } diff --git a/packages/runtime/src/model-protocol.ts b/packages/runtime/src/model-protocol.ts index 8e771cc3ff..6b902725a1 100644 --- a/packages/runtime/src/model-protocol.ts +++ b/packages/runtime/src/model-protocol.ts @@ -349,6 +349,8 @@ export interface ModelFailure { /** Provider-requested delay for the next physical attempt, in milliseconds. */ retryAfterMs?: number; code?: string; + /** Bounded, redacted provider response retained for user diagnostics. */ + diagnosticSummary?: string; } /** diff --git a/packages/runtime/src/provider-error-classification.ts b/packages/runtime/src/provider-error-classification.ts index aaa73bbe9b..91c54369a9 100644 --- a/packages/runtime/src/provider-error-classification.ts +++ b/packages/runtime/src/provider-error-classification.ts @@ -139,7 +139,7 @@ interface ProviderFailureSummary { code?: string; } -const PROVIDER_FAILURE_SUMMARY_MAX_BYTES = 2 * 1024; +export const PROVIDER_FAILURE_SUMMARY_MAX_BYTES = 2 * 1024; const PROVIDER_FAILURE_FIELD_MAX_BYTES = 256; const MAX_SAFE_TIMER_DELAY_MS = 2_147_483_647; diff --git a/packages/runtime/src/runtime-event-backfill.ts b/packages/runtime/src/runtime-event-backfill.ts index ac02be04bb..fafca4e7c1 100644 --- a/packages/runtime/src/runtime-event-backfill.ts +++ b/packages/runtime/src/runtime-event-backfill.ts @@ -470,6 +470,16 @@ function terminalRuntimeEvent(input: { ...(abortSource !== undefined ? { abortSource } : {}), }, }, + ...(status === 'failed' && turnState?.failureMessage + ? { + content: { + kind: 'error' as const, + message: turnState.failureMessage, + ...(turnState.errorClass ? { reason: turnState.errorClass } : {}), + details: { providerSummary: turnState.failureMessage }, + }, + } + : {}), ...(turnState ? { refs: { storedMessageId: turnState.id } } : {}), }, }; diff --git a/packages/runtime/src/runtime-event-read-model.ts b/packages/runtime/src/runtime-event-read-model.ts index 62ab789737..387874bf0d 100644 --- a/packages/runtime/src/runtime-event-read-model.ts +++ b/packages/runtime/src/runtime-event-read-model.ts @@ -22,6 +22,8 @@ import type { AssistantStepContentKind, StoredMessage, TurnStatus } from '@maka/ import type { RuntimeEvent, RuntimeEventStatus } from '@maka/core/runtime-event'; import type { ToolActivityKind, ToolResultContent } from '@maka/core/events'; import { markPersisted } from '@maka/core/persisted-value'; +import { truncateUtf8 } from '@maka/core/diagnostic-log'; +import { redactSecrets } from '@maka/core/redaction'; import { SANDBOX_BOUNDARY_REQUEST_STATUSES, validateSandboxBoundaryExpansion, @@ -46,6 +48,7 @@ const SETTLED_SANDBOX_BOUNDARY_STATUSES: readonly SettledSandboxBoundaryStatus[] SANDBOX_BOUNDARY_REQUEST_STATUSES.filter( (status): status is SettledSandboxBoundaryStatus => status !== 'pending', ); +const TURN_FAILURE_MESSAGE_MAX_BYTES = 256; import type { CanonicalPermissionOutcomeRecord } from './interaction-authority.js'; import { isArchivedToolResultPlaceholder } from './tool-result-archive.js'; @@ -1150,6 +1153,7 @@ function projectTerminalTurnState( const abortSource = status === 'aborted' ? abortSourceFromRuntime(event, header) : undefined; const failureClass = status === 'failed' ? failureClassFromRuntimeEvent(event, header) : undefined; + const failureMessage = status === 'failed' ? failureMessageFromRuntimeEvent(event) : undefined; const partialOutputRetained = messages.some( (message) => message.turnId === event.turnId && @@ -1172,6 +1176,7 @@ function projectTerminalTurnState( ...(status === 'aborted' ? { abortedAt: event.ts } : {}), ...(abortSource ? { abortSource } : {}), ...(status === 'failed' ? { errorClass: failureClass ?? 'unknown' } : {}), + ...(failureMessage ? { failureMessage } : {}), partialOutputRetained, }); if (failureClass === 'tool_step_cap_reached') { @@ -1202,6 +1207,17 @@ function projectTerminalTurnState( return true; } +function failureMessageFromRuntimeEvent(event: RuntimeEvent): string | undefined { + const content = event.content; + if (content?.kind !== 'error' || !content.details || Array.isArray(content.details)) { + return undefined; + } + const summary = content.details.providerSummary; + return typeof summary === 'string' && summary.length > 0 + ? truncateUtf8(redactSecrets(summary), TURN_FAILURE_MESSAGE_MAX_BYTES, '…') + : undefined; +} + function attachPendingThinking( event: RuntimeEvent, state: ProjectionState, @@ -1591,6 +1607,7 @@ function semanticMessage(message: StoredMessage): unknown { abortedAt: message.abortedAt, abortSource: message.abortSource, errorClass: message.errorClass, + failureMessage: message.failureMessage, partialOutputRetained: message.partialOutputRetained, }; case 'system_note': diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index 1da029c273..4241770141 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -2731,7 +2731,13 @@ export class RuntimeKernel implements RuntimeKernelLike { turnId: string, status: TurnRecord['status'], lineage: AgentRunLineage = {}, - options: { id?: string; ts?: number; errorClass?: string; abortSource?: string } = {}, + options: { + id?: string; + ts?: number; + errorClass?: string; + failureMessage?: string; + abortSource?: string; + } = {}, ): Promise { const ts = options.ts ?? this.deps.now(); await this.deps.store.appendMessage( @@ -2744,6 +2750,7 @@ export class RuntimeKernel implements RuntimeKernelLike { lineage, ...(options.abortSource ? { abortSource: options.abortSource } : {}), ...(options.errorClass !== undefined ? { errorClass: options.errorClass } : {}), + ...(options.failureMessage !== undefined ? { failureMessage: options.failureMessage } : {}), partialOutputRetained: await this.turnHasRetainedOutput(sessionId, turnId), }), ); diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index fe98e096f7..f100c10f08 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -4268,7 +4268,12 @@ export class SessionManager { turnId: string, status: TurnRecord['status'], lineage: AgentRunLineage = {}, - options: { ts?: number; errorClass?: string; abortSource?: string } = {}, + options: { + ts?: number; + errorClass?: string; + failureMessage?: string; + abortSource?: string; + } = {}, ): Promise { const ts = options.ts ?? this.deps.now(); await this.deps.store.appendMessage( @@ -4281,6 +4286,7 @@ export class SessionManager { lineage, ...(options.abortSource ? { abortSource: options.abortSource } : {}), ...(options.errorClass !== undefined ? { errorClass: options.errorClass } : {}), + ...(options.failureMessage !== undefined ? { failureMessage: options.failureMessage } : {}), partialOutputRetained: await this.turnHasRetainedOutput(sessionId, turnId), }), ); diff --git a/packages/runtime/src/session-projection-helpers.ts b/packages/runtime/src/session-projection-helpers.ts index 468005fc24..ba6343aa19 100644 --- a/packages/runtime/src/session-projection-helpers.ts +++ b/packages/runtime/src/session-projection-helpers.ts @@ -18,6 +18,8 @@ */ import type { AgentRunHeader } from '@maka/core/agent-run'; +import { truncateUtf8 } from '@maka/core/diagnostic-log'; +import { redactSecrets } from '@maka/core/redaction'; import { failureClassFromCompleteStopReason, type SessionEvent } from '@maka/core/events'; import type { SessionBlockedReason, @@ -46,6 +48,7 @@ export interface BuildTurnStateMessageInput { status: TurnRecord['status']; lineage?: TurnStateLineage; errorClass?: string; + failureMessage?: string; abortSource?: string; partialOutputRetained: boolean; } @@ -80,6 +83,9 @@ export function buildTurnStateMessage(input: BuildTurnStateMessageInput): TurnSt ...(input.status === 'aborted' ? { abortedAt: input.ts } : {}), ...(input.status === 'aborted' && input.abortSource ? { abortSource: input.abortSource } : {}), ...(input.status === 'failed' ? { errorClass: input.errorClass ?? 'unknown' } : {}), + ...(input.status === 'failed' && input.failureMessage + ? { failureMessage: input.failureMessage } + : {}), partialOutputRetained: input.partialOutputRetained, }; } @@ -141,12 +147,18 @@ export function statusFromEvent( export function turnStatusFromEvent( event: SessionEvent, -): { status: TurnRecord['status']; errorClass?: string } | undefined { +): { status: TurnRecord['status']; errorClass?: string; failureMessage?: string } | undefined { switch (event.type) { case 'abort': return { status: 'aborted' }; case 'error': - return { status: 'failed', errorClass: event.reason ?? event.code ?? 'unknown' }; + return { + status: 'failed', + errorClass: event.reason ?? event.code ?? 'unknown', + ...(providerFailureMessageFromEvent(event) + ? { failureMessage: providerFailureMessageFromEvent(event) } + : {}), + }; case 'complete': { if (event.stopReason === 'user_stop') return { status: 'aborted' }; const errorClass = failureClassFromCompleteStopReason(event.stopReason); @@ -158,6 +170,17 @@ export function turnStatusFromEvent( } } +/** Extract the adapter's bounded provider summary without exposing arbitrary details. */ +export function providerFailureMessageFromEvent( + event: Extract, +): string | undefined { + if (!event.details || Array.isArray(event.details)) return undefined; + const summary = event.details.providerSummary; + return typeof summary === 'string' && summary.length > 0 + ? truncateUtf8(redactSecrets(summary), 256, '…') + : undefined; +} + function blockedReasonFromErrorReason(reason: string | undefined): SessionBlockedReason { if (!reason) return 'unknown'; if (reason === 'permission_required') return 'permission_required'; diff --git a/packages/ui/src/__tests__/chat-turn-steering-order.test.ts b/packages/ui/src/__tests__/chat-turn-steering-order.test.ts index 2583e6c095..b871fd625e 100644 --- a/packages/ui/src/__tests__/chat-turn-steering-order.test.ts +++ b/packages/ui/src/__tests__/chat-turn-steering-order.test.ts @@ -71,3 +71,31 @@ test('renders steering where it arrived in the assistant timeline', () => { assert.equal(visibleText.split(text).length - 1, 1, `${text} should render exactly once`); } }); + +test('renders provider failure detail collapsed beneath the classified banner', () => { + const turn: TurnViewModel = { + turnId: 'turn-provider-failure', + status: 'failed', + partialOutputRetained: false, + user: { id: 'user', role: 'user', text: 'request', ts: 1 }, + tools: [], + notes: [], + startedAt: 1, + timeline: [{ kind: 'text', text: 'partial output', messageId: 'assistant', ts: 2 }], + }; + const markup = renderToStaticMarkup( + createElement(LocaleProvider, { + locale: 'en', + children: createElement(TurnView, { + turn, + failedReasonLabel: 'Rate limit exceeded', + failedDiagnostic: '429 rate limit (code=rate_limit_exceeded, requestId=req-123)', + }), + }), + ); + assert.match(markup, /]*maka-turn-failed-diagnostic/); + assert.match(markup, /Provider response details<\/summary>/); + assert.match(markup, /429 rate limit/); + assert.doesNotMatch(markup, /]*open/); + assert.ok(markup.indexOf('Rate limit exceeded') < markup.indexOf('429 rate limit')); +}); diff --git a/packages/ui/src/__tests__/materialize.test.ts b/packages/ui/src/__tests__/materialize.test.ts index 179be9ea18..ed83bc818a 100644 --- a/packages/ui/src/__tests__/materialize.test.ts +++ b/packages/ui/src/__tests__/materialize.test.ts @@ -485,6 +485,24 @@ describe("unfinished tools take their status from the turn", () => { ]); assert.equal(turn?.tools[0]?.status, "interrupted"); }); + + test("restores provider failure detail from a reloaded turn state", () => { + const [turn] = materializeTurns([ + userMsg("t1", 1, "run it"), + { + type: "turn_state", + id: "s1", + turnId: "t1", + ts: 2, + status: "failed", + errorClass: "rate_limit", + failureMessage: "provider says retry after 30 seconds", + partialOutputRetained: false, + }, + ]); + assert.equal(turn?.errorClass, "rate_limit"); + assert.equal(turn?.failureMessage, "provider says retry after 30 seconds"); + }); }); describe("live tool status over persisted", () => { diff --git a/packages/ui/src/chat-turn.tsx b/packages/ui/src/chat-turn.tsx index 1fedef9546..4c38b26b53 100644 --- a/packages/ui/src/chat-turn.tsx +++ b/packages/ui/src/chat-turn.tsx @@ -396,6 +396,8 @@ export const TurnView = memo(function TurnView(props: { * is the outcome, this is the execution state, and both can be true. */ failedExecutionStateLabel?: string; + /** Bounded provider response detail shown collapsed beneath the category. */ + failedDiagnostic?: string; safeResumeAction?: { pending: boolean; detail?: string; @@ -682,35 +684,43 @@ export const TurnView = memo(function TurnView(props: { is one — it explains why the button did nothing, which outranks execution state on the one turn that can have both. */} {ownsTurnChrome && turn.status === 'failed' && props.failedReasonLabel && ( - - ), - } - : {})} - /> + <> + + ), + } + : {})} + /> + {props.failedDiagnostic && ( +
+ {copy.failedDiagnostic} +
{props.failedDiagnostic}
+
+ )} + )} {ownsTurnChrome && props.liveStreaming && ( <> diff --git a/packages/ui/src/chat-view.tsx b/packages/ui/src/chat-view.tsx index ea812fe1be..2b220dcc3b 100644 --- a/packages/ui/src/chat-view.tsx +++ b/packages/ui/src/chat-view.tsx @@ -788,6 +788,9 @@ export function ChatView(props: { failedExecutionStateLabel={ turnPresentation?.failedExecutionStateLabels[turn.turnId] } + failedDiagnostic={turnPresentation?.failedReasonLabels[turn.turnId] + ? turn.failureMessage + : undefined} safeResumeAction={turnPresentation?.resumeCandidateTurnId === turn.turnId ? props.safeResumeAction : undefined} diff --git a/packages/ui/src/conversation-copy.ts b/packages/ui/src/conversation-copy.ts index 98ae0115c0..6f77e41096 100644 --- a/packages/ui/src/conversation-copy.ts +++ b/packages/ui/src/conversation-copy.ts @@ -280,6 +280,7 @@ export interface ConversationCopy { providerRetryReason: Record; safeResumePending: string; safeResume: string; + failedDiagnostic: string; thinking: string; truncated: string; copied: string; @@ -525,6 +526,7 @@ const CONVERSATION_COPY = { }, messages: { you: '你', assistant: 'Maka', processing: '正在处理…', continuing: '继续中…', workingPhrases: ['正在琢磨…', '正在推敲…', '正在盘算…', '正在钻研…', '正在忙活…', '正在梳理…', '正在打磨…', '正在鼓捣…', '正在酝酿…', '正在攻坚…', '正在权衡…', '正在拾掇…'], providerRetryScheduled: (seconds, attempt, maxAttempts) => `${formatRetryDelay(seconds, 'zh')}后重试(${attempt}/${maxAttempts})`, providerRetryStarted: (attempt, maxAttempts) => `正在重试(${attempt}/${maxAttempts})`, providerRetryWaiting: (attempt, maxAttempts) => `等待重试(${attempt}/${maxAttempts})`, providerRetryReason: { network: '网络中断', provider_capacity: '模型服务暂时满载', provider_unavailable: '模型服务暂时不可用', rate_limit: '触发模型速率限制', timeout: '请求超时', unknown: '模型请求失败' }, safeResumePending: '正在检查…', safeResume: '继续这一轮', thinking: '深度思考', truncated: '已截断', copied: '已复制', copying: '复制中', copyFailed: '复制失败', copy: '复制', editMessage: '编辑并重发', editMessageDisabledRunning: '当前回答仍在进行中,结束后再编辑', editMessageDisabledAttachments: '包含附件的历史消息暂不支持编辑并重发', editMessageDisabledQuotes: '包含引用的历史消息暂不支持编辑并重发', editMessageDisabledTransformedText: '包含已展开上下文的历史消息暂不支持编辑并重发', + failedDiagnostic: 'Provider 响应详情', editMessageDisabledDirectoryReferences: '包含文件夹引用的历史消息暂不支持编辑并重发', userAriaLabel: '你发送的消息', systemAriaLabel: '系统消息', assistantAriaLabel: 'Maka 的回答', answerActionsAriaLabel: (context) => `回答操作${context ? `:${context}` : ''}`, answerActionAriaLabel: (action, context) => `${action}回答${context ? `:${context}` : ''}`, messageActionAriaLabel: (action, context) => `${action}消息${context ? `:${context}` : ''}`, sourceAriaLabel: '本轮回答的来源', derivativesAriaLabel: '本轮回答的衍生', scheduledTaskTriggered: '定时任务触发', scheduledTaskTitle: (id) => `由定时任务触发 · ${id}`, legacyAutomationTriggered: '旧版自动化(仅历史)', legacyAutomationTitle: (id) => `由旧版自动化触发 · ${id} · 仅保留历史,不会再次执行`, goalContinued: 'Goal 自动继续', goalTitle: (id) => `由 Goal 继续执行 · ${id}`, agentGraphTriggered: 'Agent Graph 自动继续', agentGraphTitle: (graphId) => `由 Agent Graph 调度器触发 · ${graphId}`, thinkingTruncatedTitle: '部分 reasoning 已截断;显示的是最近的内容', outputTruncatedTitle: '助手输出已超过单次回合上限,超出部分未渲染。如需完整内容请重新生成或查看持久化的任务日志。', removeAttachmentAriaLabel: (name) => `移除 ${name}`, quoteLabel: '引用', quoteExpandAriaLabel: '展开引用全文', quoteCollapseAriaLabel: '收起引用', removeQuoteAriaLabel: '移除引用', aborted: '已中断', abortedByStop: '已中断 · 由停止按钮触发', @@ -683,6 +685,7 @@ const CONVERSATION_COPY = { }, messages: { you: 'You', assistant: 'Maka', processing: 'Working…', continuing: 'Continuing…', workingPhrases: ['Pondering…', 'Tinkering…', 'Untangling…', 'Digging in…', 'Mulling…', 'Chewing on it…', 'Wrangling…', 'Piecing it together…'], providerRetryScheduled: (seconds, attempt, maxAttempts) => `Retrying in ${formatRetryDelay(seconds, 'en')} (${attempt}/${maxAttempts})`, providerRetryStarted: (attempt, maxAttempts) => `Retrying (${attempt}/${maxAttempts})`, providerRetryWaiting: (attempt, maxAttempts) => `Waiting to retry (${attempt}/${maxAttempts})`, providerRetryReason: { network: 'Network interrupted', provider_capacity: 'The model service is temporarily at capacity', provider_unavailable: 'Model service temporarily unavailable', rate_limit: 'Model rate limit reached', timeout: 'Request timed out', unknown: 'Model request failed' }, safeResumePending: 'Checking…', safeResume: 'Continue this turn', thinking: 'Thinking', truncated: 'Truncated', copied: 'Copied', copying: 'Copying', copyFailed: 'Copy failed', copy: 'Copy', editMessage: 'Edit & resend', editMessageDisabledRunning: 'Wait for this answer to finish before editing', editMessageDisabledAttachments: 'Edit & resend does not yet support messages with attachments', editMessageDisabledQuotes: 'Edit & resend does not yet support messages with quotes', editMessageDisabledTransformedText: 'Edit & resend does not yet support messages with expanded context', + failedDiagnostic: 'Provider response details', editMessageDisabledDirectoryReferences: 'Edit & resend does not yet support messages with folder references', userAriaLabel: 'Your message', systemAriaLabel: 'System message', assistantAriaLabel: "Maka's response", answerActionsAriaLabel: (context) => `Response actions${context ? `: ${context}` : ''}`, answerActionAriaLabel: (action, context) => `${action} response${context ? `: ${context}` : ''}`, messageActionAriaLabel: (action, context) => `${action} message${context ? `: ${context}` : ''}`, sourceAriaLabel: 'Source of this response', derivativesAriaLabel: 'Responses derived from this one', scheduledTaskTriggered: 'Triggered by scheduled task', scheduledTaskTitle: (id) => `Triggered by scheduled task · ${id}`, legacyAutomationTriggered: 'Legacy Automation (history only)', legacyAutomationTitle: (id) => `Triggered by legacy Automation · ${id} · Historical only; it will not run again`, goalContinued: 'Continued by Goal', goalTitle: (id) => `Continued by Goal · ${id}`, agentGraphTriggered: 'Continued by Agent Graph', agentGraphTitle: (graphId) => `Triggered by the Agent Graph scheduler · ${graphId}`, thinkingTruncatedTitle: 'Some reasoning was truncated; showing the most recent content', outputTruncatedTitle: 'The assistant output exceeded the per-turn limit. Regenerate it or inspect the persisted task log for the complete content.', removeAttachmentAriaLabel: (name) => `Remove ${name}`, quoteLabel: 'Quote', quoteExpandAriaLabel: 'Show the full quoted excerpt', quoteCollapseAriaLabel: 'Collapse the quoted excerpt', removeQuoteAriaLabel: 'Remove quote', aborted: 'Interrupted', abortedByStop: 'Interrupted · Stop button', diff --git a/packages/ui/src/materialize.ts b/packages/ui/src/materialize.ts index 02dc73fde9..4ab0b82570 100644 --- a/packages/ui/src/materialize.ts +++ b/packages/ui/src/materialize.ts @@ -376,6 +376,7 @@ export interface TurnViewModel { abortedAt?: number; abortSource?: string; errorClass?: string; + failureMessage?: string; partialOutputRetained: boolean; user?: ChatItem; tools: ToolActivityItem[]; @@ -703,6 +704,7 @@ export function materializeTurns( : {}), ...(record?.abortSource ? { abortSource: record.abortSource } : {}), ...(record?.errorClass ? { errorClass: record.errorClass } : {}), + ...(record?.failureMessage ? { failureMessage: record.failureMessage } : {}), partialOutputRetained: record?.partialOutputRetained ?? false, tools: [], notes: [], diff --git a/packages/ui/src/styles.css b/packages/ui/src/styles.css index 659d9415f7..3f341f9fa9 100644 --- a/packages/ui/src/styles.css +++ b/packages/ui/src/styles.css @@ -790,6 +790,19 @@ /* Astryx `Banner` paints the failed-turn surface; this only sets the block rhythm between it and the timeline entry above it. */ .maka-turn-failed-banner { margin-block: var(--space-1) var(--space-0-5); } +.maka-turn-failed-diagnostic { + margin-block: var(--space-0-5) var(--space-1); + padding-inline: var(--space-1); + color: var(--muted-foreground); + font-size: 0.8rem; +} +.maka-turn-failed-diagnostic summary { cursor: pointer; } +.maka-turn-failed-diagnostic pre { + margin-block: var(--space-0-5) 0; + white-space: pre-wrap; + overflow-wrap: anywhere; + font: inherit; +} .maka-turn-lineage-row, .maka-turn-footer {