Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion packages/core/src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -930,6 +930,8 @@ export interface TurnStateMessage {
/** Diagnostic source for user/renderer-triggered aborts, e.g. renderer.stop_button. */
abortSource?: string;
errorClass?: string;
/** Bounded provider response summary for a failed turn. */
failureMessage?: string;
partialOutputRetained: boolean;
}

Expand Down Expand Up @@ -1056,6 +1058,8 @@ export interface TurnRecord {
abortedAt?: number;
abortSource?: string;
errorClass?: string;
/** Bounded provider response summary for a failed turn. */
failureMessage?: string;
partialOutputRetained: boolean;
}

Expand Down Expand Up @@ -1163,6 +1167,7 @@ const TURN_STATE_MESSAGE_SHAPE = defineObjectShape<TurnStateMessage>()(
'abortedAt',
'abortSource',
'errorClass',
'failureMessage',
],
);
const WORKHUB_DELEGATION_ASSIGNED_MESSAGE_SHAPE =
Expand Down Expand Up @@ -1409,7 +1414,8 @@ function decodeMessage(
isOptionalString(message.parentSessionId) &&
(message.abortedAt === undefined || isFiniteNumber(message.abortedAt)) &&
isOptionalString(message.abortSource) &&
isOptionalString(message.errorClass)
isOptionalString(message.errorClass) &&
isOptionalString(message.failureMessage)
)
return message as unknown as TurnStateMessage;
break;
Expand Down Expand Up @@ -1656,6 +1662,7 @@ export function deriveTurnRecords(messages: readonly StoredMessage[]): TurnRecor
...(latestState.abortedAt !== undefined ? { abortedAt: latestState.abortedAt } : {}),
...(latestState.abortSource ? { abortSource: latestState.abortSource } : {}),
...(latestState.errorClass ? { errorClass: latestState.errorClass } : {}),
...(latestState.failureMessage ? { failureMessage: latestState.failureMessage } : {}),
partialOutputRetained: latestState.partialOutputRetained || partialOutputRetained,
};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,9 @@ test('projects a failed Turn message from the canonical terminal event', async (
recoverable: false,
code: 'provider_error',
message: 'canonical provider failure api_key=sk-test-secret-value',
details: {
providerSummary: 'provider rejected model (code=provider_error, requestId=req-123)',
},
},
context,
memory,
Expand Down Expand Up @@ -417,7 +420,7 @@ test('projects a failed Turn message from the canonical terminal event', async (
if (canonical?.rootTurn?.status === 'failed') {
assert.equal(
canonical.rootTurn.failureMessage,
'canonical provider failure api_key=[redacted]',
'provider rejected model (code=provider_error, requestId=req-123)',
);
}
});
Expand Down
42 changes: 42 additions & 0 deletions packages/runtime-host/src/__tests__/session-turns.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
projectSessionTurnContribution,
projectSessionTurnContributionForWire,
SESSION_TURN_DIAGNOSTIC_MAX_BYTES,
SESSION_TURN_FAILURE_MESSAGE_MAX_BYTES,
SESSION_TURN_LANDMARK_RESULT_MAX_BYTES,
} from '../protocol/session-turns.js';

Expand Down Expand Up @@ -110,6 +111,47 @@ test('bounds turn diagnostics before publishing a contribution', () => {
);
});

test('persists and bounds the provider failure summary in the Turn projection', () => {
const contribution = projectSessionTurnContributionForWire({
turnId: 'turn-1',
firstSequence: 0,
latestState: {
sequence: 0,
message: {
type: 'turn_state',
id: 'state-1',
turnId: 'turn-1',
ts: 1,
status: 'failed',
partialOutputRetained: false,
errorClass: 'rate_limit',
failureMessage: `provider says ${'x'.repeat(10_000)}`,
},
},
userPromptPreview: null,
hasAssistantMessage: false,
hasAssistantOutput: false,
hasToolResult: false,
hasFailedToolResult: false,
hasAbortNote: false,
});

assert.ok(
Buffer.byteLength(contribution.latestState!.message.failureMessage!, 'utf8') <=
SESSION_TURN_FAILURE_MESSAGE_MAX_BYTES,
);
const decoded = decodeSessionTurnsQueryResult({
sessionId: 'session-1',
throughSequence: 0,
contributions: [contribution],
nextPosition: null,
});
assert.equal(
decoded.contributions[0]!.latestState!.message.failureMessage,
contribution.latestState!.message.failureMessage,
);
});

test('rejects invalid turn-state references before publishing a contribution', () => {
assert.throws(() =>
projectSessionTurnContributionForWire({
Expand Down
9 changes: 7 additions & 2 deletions packages/runtime-host/src/adapter/session-projector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,10 @@ export class RuntimeHostSessionProjector {
ts: terminal.ts,
recoverable: false,
reason,
message: `Turn failed: ${reason}`,
message: terminal.failureMessage ?? `Turn failed: ${reason}`,
...(terminal.failureMessage
? { details: { providerSummary: terminal.failureMessage } }
: {}),
});
} else {
events.push({
Expand Down Expand Up @@ -329,7 +332,8 @@ export class RuntimeHostSessionProjector {
ts,
recoverable: false,
reason,
message: `Turn failed: ${reason}`,
message: turn.failureMessage ?? `Turn failed: ${reason}`,
...(turn.failureMessage ? { details: { providerSummary: turn.failureMessage } } : {}),
},
];
}
Expand Down Expand Up @@ -483,6 +487,7 @@ export class RuntimeHostSessionProjector {
recoverable: false,
reason: root.failureClass,
message: root.failureMessage ?? `Turn failed: ${root.failureClass}`,
...(root.failureMessage ? { details: { providerSummary: root.failureMessage } } : {}),
});
} else {
events.push({
Expand Down
5 changes: 4 additions & 1 deletion packages/runtime-host/src/protocol/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,10 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const;
export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const;
// Increment when the same protocol version no longer guarantees safe Client-Host
// interoperability. Mismatches are rejected before domain commands are admitted.
export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 101 as const;
export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 102 as const;
// 102: Session Turn projections carry bounded provider failure summaries for
// live and reloaded failed-turn diagnostics. Older peers cannot preserve or
// render this additional failure context safely.
// 101: Session Turn requests can carry regeneration intents and Guests can
// atomically withdraw pending requests. Older peers do not share this command
// vocabulary or the expanded Guest operation grant.
Expand Down
17 changes: 17 additions & 0 deletions packages/runtime-host/src/protocol/session-turns.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import { defineOperation } from './operation-spec.js';
export const SESSION_TURN_QUERY_MAX_CONTRIBUTIONS = 128;
export const SESSION_TURN_QUERY_RESULT_MAX_BYTES = 192 * 1024;
export const SESSION_TURN_DIAGNOSTIC_MAX_BYTES = 128;
export const SESSION_TURN_FAILURE_MESSAGE_MAX_BYTES = 256;
export const SESSION_TURN_PROMPT_PREVIEW_MAX_BYTES = 256;
export const SESSION_TURN_LANDMARK_MAX_ITEMS = 64;
export const SESSION_TURN_LANDMARK_LABEL_MAX_BYTES = 96;
Expand Down Expand Up @@ -179,6 +180,14 @@ function projectTurnStateMessageForWire(message: TurnStateMessage): TurnStateMes
...(message.errorClass
? { errorClass: truncateUtf8(message.errorClass, SESSION_TURN_DIAGNOSTIC_MAX_BYTES) }
: {}),
...(message.failureMessage
? {
failureMessage: truncateUtf8(
message.failureMessage,
SESSION_TURN_FAILURE_MESSAGE_MAX_BYTES,
),
}
: {}),
partialOutputRetained: message.partialOutputRetained,
};
}
Expand All @@ -205,6 +214,7 @@ export function projectSessionTurnContribution(contribution: SessionTurnContribu
...(state.abortedAt !== undefined ? { abortedAt: state.abortedAt } : {}),
...(state.abortSource ? { abortSource: state.abortSource } : {}),
...(state.errorClass ? { errorClass: state.errorClass } : {}),
...(state.failureMessage ? { failureMessage: state.failureMessage } : {}),
partialOutputRetained: state.partialOutputRetained || partialOutputRetained,
};
}
Expand Down Expand Up @@ -424,6 +434,13 @@ function decodeSessionTurnContribution(value: unknown): SessionTurnContribution
SESSION_TURN_DIAGNOSTIC_MAX_BYTES,
);
}
if (message.failureMessage !== undefined) {
requireUtf8String(
message.failureMessage,
'Session turn failure message',
SESSION_TURN_FAILURE_MESSAGE_MAX_BYTES,
);
}
latestState = {
sequence: requireCount(state.sequence, 'Session turn state sequence'),
message,
Expand Down
13 changes: 12 additions & 1 deletion packages/runtime-host/src/server/canonical-turn-snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,10 @@ export async function readCanonicalTurnSnapshot(
const failureMessage =
fact.terminalEvent.content?.kind === 'error'
? truncateUtf8(
redactSecrets(fact.terminalEvent.content.message),
redactSecrets(
providerFailureSummaryFromRuntimeEvent(fact.terminalEvent) ??
fact.terminalEvent.content.message,
),
TURN_FAILURE_MESSAGE_MAX_BYTES,
'…',
)
Expand Down Expand Up @@ -110,6 +113,14 @@ export async function readCanonicalTurnSnapshot(
return { sessionId, turnId, runId, status: run.status };
}

function providerFailureSummaryFromRuntimeEvent(
event: import('@maka/core/runtime-event').RuntimeEvent,
): string | undefined {
const details = event.content?.kind === 'error' ? event.content.details : undefined;
if (!details || Array.isArray(details)) return undefined;
const summary = details.providerSummary;
return typeof summary === 'string' && summary.length > 0 ? summary : undefined;
}
function readContextCompactionOutcome(value: unknown): ContextCompactionOutcome | undefined {
if (!value || typeof value !== 'object') return undefined;
const outcome = value as Record<string, unknown>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ export function projectSharedSessionTranscriptMessage(
...(message.abortedAt === undefined ? {} : { abortedAt: message.abortedAt }),
...(message.abortSource === undefined ? {} : { abortSource: message.abortSource }),
...(message.errorClass === undefined ? {} : { errorClass: message.errorClass }),
...(message.failureMessage === undefined ? {} : { failureMessage: message.failureMessage }),
partialOutputRetained: message.partialOutputRetained,
};
case 'token_usage':
Expand Down
6 changes: 6 additions & 0 deletions packages/runtime/src/__tests__/model-adapter-onerror.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ describe('ModelAdapter.startStream onError', () => {
message: 'Rate limit exceeded',
retryable: true,
retryAfterMs: 2500,
diagnosticSummary: 'rate limited (status=429)',
},
]);
const outcome = await requireAlreadySettled(result.outcome);
Expand Down Expand Up @@ -204,6 +205,7 @@ describe('ModelAdapter.startStream onError', () => {
kind: 'unknown',
message: 'Plaintext Responses reasoning item is missing final summary metadata',
retryable: false,
diagnosticSummary: 'Plaintext Responses reasoning item is missing final summary metadata',
},
]);
const outcome = await requireAlreadySettled(result.outcome);
Expand Down Expand Up @@ -249,6 +251,7 @@ describe('ModelAdapter.startStream onError', () => {
retryable: false,
message: 'Rate limit exceeded',
code: 'rate_limit_exceeded',
diagnosticSummary: 'Provider stopped the stream with an error (code=rate_limit_exceeded)',
});
assert.equal(outcome.usage?.rawFinishReason, 'rate_limit_exceeded');
});
Expand Down Expand Up @@ -288,6 +291,7 @@ describe('ModelAdapter.startStream onError', () => {
message: 'Rate limit exceeded',
retryable: true,
retryAfterMs: 2500,
diagnosticSummary: 'rate limited (status=429)',
},
]);
assert.deepEqual(await result.outcome, {
Expand Down Expand Up @@ -485,6 +489,8 @@ describe('ModelAdapter.startStream onError', () => {
kind: 'network',
message: 'Network error',
retryable: true,
diagnosticSummary:
'Client network socket disconnected before secure TLS connection was established',
},
]);
} finally {
Expand Down
22 changes: 22 additions & 0 deletions packages/runtime/src/__tests__/model-adapter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -378,13 +378,15 @@ describe('ModelAdapter stream and error normalization', () => {
code: '429',
message: 'Rate limit exceeded',
retryable: false,
diagnosticSummary: '429 rate limit (code=429)',
});
// The backend consumes the typed failure without recovering the raw
// provider error shape.
const shaped = adapter.makeErrorEvent('turn-1', errorEvent.failure);
assert.equal(shaped.reason, 'rate_limit');
assert.equal(shaped.code, '429');
assert.equal(shaped.message, 'Rate limit exceeded');
assert.deepEqual(shaped.details, { providerSummary: '429 rate limit (code=429)' });
});

test('normalizes a status-less provider server_error into a retryable outage', () => {
Expand All @@ -408,6 +410,8 @@ describe('ModelAdapter stream and error normalization', () => {
code: 'server_error',
message: 'Provider returned an error',
retryable: true,
diagnosticSummary:
'Streaming response failed: [502] Upstream error from Nvidia: Service temporarily overloaded (code=server_error)',
},
});
});
Expand Down Expand Up @@ -842,6 +846,24 @@ describe('ModelAdapter stream and error normalization', () => {
assert.equal(event.message.includes('sk-live-secret-token-value'), false);
});

test('re-scrubs a pre-normalized provider diagnostic at the event boundary', () => {
const event = newAdapter().makeErrorEvent('turn-1', {
type: 'model_failure',
kind: 'rate_limit',
message: 'Rate limit exceeded',
retryable: false,
diagnosticSummary: `provider says api_key=sk-live-secret-token-value ${'x'.repeat(4_000)}`,
});

const providerSummary =
event.details && !Array.isArray(event.details) ? event.details.providerSummary : undefined;
assert.equal(
typeof providerSummary === 'string' && providerSummary.includes('sk-live-secret-token-value'),
false,
);
assert.equal(Buffer.byteLength(String(providerSummary ?? ''), 'utf8') <= 2 * 1024, true);
});

test('normalizes cache and reasoning usage variants in the adapter module', () => {
assert.deepEqual(
normalizeAiSdkUsage({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,8 @@ describe('OpenAI Responses ModelAdapter continuation', () => {
retryable: true,
code: 'OPENAI_RESPONSES_WEBSOCKET_TRANSPORT_ERROR',
message: 'Network error',
diagnosticSummary:
'closed before completion (code=OPENAI_RESPONSES_WEBSOCKET_TRANSPORT_ERROR)',
},
]);
});
Expand Down
30 changes: 30 additions & 0 deletions packages/runtime/src/__tests__/runtime-event-read-model.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1603,6 +1603,36 @@ describe('projectRuntimeEventsToStoredMessages', () => {
assert.deepStrictEqual(out.diagnostics, []);
});

test('carries the bounded provider summary into the durable failed turn', () => {
const out = projectRuntimeEventsToStoredMessages(
[
ev({
id: 'evt-provider-failed',
ts: ts + 9,
status: 'failed',
content: {
kind: 'error',
reason: 'rate_limit',
message: 'Rate limit exceeded',
details: {
providerSummary: `provider says api_key=sk-live-secret ${'x'.repeat(1_000)}`,
},
},
actions: { endInvocation: true },
}),
],
{
runHeaders: [{ ...header, status: 'failed', failureClass: 'rate_limit' }],
},
);

const state = out.messages.find((message) => message.type === 'turn_state');
assert.equal(state?.type, 'turn_state');
assert.equal(state?.errorClass, 'rate_limit');
assert.equal(state?.failureMessage?.includes('sk-live-secret'), false);
assert.ok(Buffer.byteLength(state?.failureMessage ?? '', 'utf8') <= 256);
});

test('tool step cap terminal fact projects a persistent system notice', () => {
const out = projectRuntimeEventsToStoredMessages(
[
Expand Down
Loading